class SvcMonitor(object):

    """
    data + methods used/referred to by ssrc and arc greenlets
    """
    _REACTION_MAP = {
        "service_appliance_set": {
            'self': [],
            'service_appliance': []
        },
        "service_appliance": {
            'self': ['service_appliance_set','physical_interface'],
            'service_appliance_set': []
        },
        "loadbalancer_pool": {
            'self': [],
            'virtual_ip': [],
            'loadbalancer_member': [],
            'loadbalancer_healthmonitor': [],
        },
        "loadbalancer_member": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "virtual_ip": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "loadbalancer_healthmonitor": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "service_instance": {
            'self': ['virtual_machine','virtual_machine_interface'],
            'virtual_machine': [],
            'virtual_machine_interface' : []
        },
        "instance_ip": {
            'self': [],
        },
        "floating_ip": {
            'self': [],
        },
        "service_template": {
            'self': [],
        },
        "physical_router": {
            'self': [],
        },
        "physical_interface": {
            'self': [],
            'service_appliance':['virtual_machine_interface'],
            'virtual_machine_interfaces':['service_appliance'],
        },
        "logical_interface": {
            'self': [],
        },
        "logical_router": {
            'self': [],
        },
        "virtual_network": {
            'self': [],
        },
        "virtual_machine": {
            'self': ['virtual_machine_interface'],
            'service_instance': [],
            'virtual_machine_interface': [],
        },
        "virtual_machine_interface": {
            'self': ['interface_route_table', 'virtual_machine'],
            'interface_route_table': [],
            'virtual_machine': [],
            'service_instance': ['physical_interface'],
            'physical_interface': ['service_instance']
        },
        "interface_route_table": {
            'self': [],
            'virtual_machine_interface': [],
        },
        "project": {
            'self': [],
        },
        "logical_router": {
            'self': [],
        },
    }

    def __init__(self, args=None):
        self._args = args

        # initialize discovery client
        self._disc = None
        if self._args.disc_server_ip and self._args.disc_server_port:
            self._disc = client.DiscoveryClient(
                self._args.disc_server_ip,
                self._args.disc_server_port,
                ModuleNames[Module.SVC_MONITOR])
        # initialize logger
        self.logger = ServiceMonitorLogger(self._disc, args)

        # rotating log file for catchall errors
        self._err_file = self._args.trace_file
        self._svc_err_logger = logging.getLogger('SvcErrLogger')
        self._svc_err_logger.setLevel(logging.ERROR)
        try:
            with open(self._err_file, 'a'):
                handler = logging.handlers.RotatingFileHandler(
                    self._err_file, maxBytes=64*1024, backupCount=2)
                self._svc_err_logger.addHandler(handler)
        except IOError:
            self.logger.log_warning("Failed to open trace file %s" %
                                    self._err_file)

        # Connect to Rabbit and Initialize cassandra connection
        self._connect_rabbit()

    def _connect_rabbit(self):
        rabbit_server = self._args.rabbit_server
        rabbit_port = self._args.rabbit_port
        rabbit_user = self._args.rabbit_user
        rabbit_password = self._args.rabbit_password
        rabbit_vhost = self._args.rabbit_vhost
        rabbit_ha_mode = self._args.rabbit_ha_mode

        self._db_resync_done = gevent.event.Event()

        q_name = 'svc_mon.%s' % (socket.gethostname())
        self._vnc_kombu = VncKombuClient(rabbit_server, rabbit_port,
                                         rabbit_user, rabbit_password,
                                         rabbit_vhost, rabbit_ha_mode,
                                         q_name, self._vnc_subscribe_callback,
                                         self.logger.log)

        self._cassandra = ServiceMonitorDB(self._args, self.logger)
        DBBaseSM.init(self, self.logger, self._cassandra)
    # end _connect_rabbit

    def _vnc_subscribe_callback(self, oper_info):
        self._db_resync_done.wait()
        try:
            self._vnc_subscribe_actions(oper_info)
        except Exception:
            cgitb_error_log(self)

    def _vnc_subscribe_actions(self, oper_info):
        try:
            msg = "Notification Message: %s" % (pformat(oper_info))
            self.logger.log_debug(msg)
            obj_type = oper_info['type'].replace('-', '_')
            obj_class = DBBaseSM.get_obj_type_map().get(obj_type)
            if obj_class is None:
                return

            if oper_info['oper'] == 'CREATE' or oper_info['oper'] == 'UPDATE':
                dependency_tracker = DependencyTracker(
                    DBBaseSM.get_obj_type_map(), self._REACTION_MAP)
                obj_id = oper_info['uuid']
                obj = obj_class.get(obj_id)
                if obj is not None:
                    dependency_tracker.evaluate(obj_type, obj)
                else:
                    obj = obj_class.locate(obj_id)
                obj.update()
                dependency_tracker.evaluate(obj_type, obj)
            elif oper_info['oper'] == 'DELETE':
                obj_id = oper_info['uuid']
                obj = obj_class.get(obj_id)
                if obj is None:
                    return
                dependency_tracker = DependencyTracker(
                    DBBaseSM.get_obj_type_map(), self._REACTION_MAP)
                dependency_tracker.evaluate(obj_type, obj)
                obj_class.delete(obj_id)
            else:
                # unknown operation
                self.logger.log_error('Unknown operation %s' %
                                      oper_info['oper'])
                return

            if obj is None:
                self.logger.log_error('Error while accessing %s uuid %s' % (
                                      obj_type, obj_id))
                return

        except Exception:
            cgitb_error_log(self)

        for sas_id in dependency_tracker.resources.get(
                'service_appliance_set', []):
            sas_obj = ServiceApplianceSetSM.get(sas_id)
            if sas_obj is not None:
                sas_obj.add()

        for lb_pool_id in dependency_tracker.resources.get(
                'loadbalancer_pool', []):
            lb_pool = LoadbalancerPoolSM.get(lb_pool_id)
            if lb_pool is not None:
                lb_pool.add()

        for si_id in dependency_tracker.resources.get('service_instance', []):
            si = ServiceInstanceSM.get(si_id)
            if si:
                si.state = 'launch'
                self._create_service_instance(si)
            else:
                self.logger.log_info("Deleting SI %s" % si_id)
                for vm_id in dependency_tracker.resources.get(
                        'virtual_machine', []):
                    vm = VirtualMachineSM.get(vm_id)
                    self._delete_service_instance(vm)
                self.logger.log_info("SI %s deletion succeed" % si_id)

        for vn_id in dependency_tracker.resources.get('virtual_network', []):
            vn = VirtualNetworkSM.get(vn_id)
            if vn:
                for si_id in ServiceInstanceSM:
                    si = ServiceInstanceSM.get(si_id)
                    intf_list = []
                    if si.params:
                        intf_list = si.params.get('interface_list', [])
                    for intf in intf_list:
                        if (':').join(vn.fq_name) in intf.values():
                            self._create_service_instance(si)

        for vmi_id in dependency_tracker.resources.get(
                'virtual_machine_interface', []):
            vmi = VirtualMachineInterfaceSM.get(vmi_id)
            if vmi:
                for vm_id in dependency_tracker.resources.get(
                        'virtual_machine', []):
                    vm = VirtualMachineSM.get(vm_id)
                    self.port_delete_or_si_link(vm, vmi)
            else:
                for irt_id in dependency_tracker.resources.get(
                        'interface_route_table', []):
                    self._delete_interface_route_table(irt_id)

        for fip_id in dependency_tracker.resources.get('floating_ip', []):
            fip = FloatingIpSM.get(fip_id)
            if fip:
                for vmi_id in fip.virtual_machine_interfaces:
                    vmi = VirtualMachineInterfaceSM.get(vmi_id)
                    if vmi and vmi.virtual_ip:
                        self.netns_manager.add_fip_to_vip_vmi(vmi, fip)

        for lr_id in dependency_tracker.resources.get('logical_router', []):
            lr = LogicalRouterSM.get(lr_id)
            if lr:
                self.snat_agent.update_snat_instance(lr)

    def post_init(self, vnc_lib, args=None):
        # api server
        self._vnc_lib = vnc_lib

        self._nova_client = importutils.import_object(
            'svc_monitor.nova_client.ServiceMonitorNovaClient',
            self._args, self.logger)

        # agent manager
        self._agent_manager = AgentManager()

        # load vrouter scheduler
        self.vrouter_scheduler = importutils.import_object(
            self._args.si_netns_scheduler_driver,
            self._vnc_lib, self._nova_client,
            self._args)

        # load virtual machine instance manager
        self.vm_manager = importutils.import_object(
            'svc_monitor.virtual_machine_manager.VirtualMachineManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load network namespace instance manager
        self.netns_manager = importutils.import_object(
            'svc_monitor.instance_manager.NetworkNamespaceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load a vrouter instance manager
        self.vrouter_manager = importutils.import_object(
            'svc_monitor.vrouter_instance_manager.VRouterInstanceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client,
            self._agent_manager, self._args)

        # load PNF instance manager
        self.ps_manager = importutils.import_object(
            'svc_monitor.physical_service_manager.PhysicalServiceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client,
            self._agent_manager, self._args)

        # load a loadbalancer agent
        self.loadbalancer_agent = LoadbalancerAgent(
            self, self._vnc_lib,
            self._cassandra, self._args)
        self._agent_manager.register_agent(self.loadbalancer_agent)

        # load a snat agent
        self.snat_agent = SNATAgent(self, self._vnc_lib,
                                    self._cassandra, self._args)
        self._agent_manager.register_agent(self.snat_agent)

        # Read the cassandra and populate the entry in ServiceMonitor DB
        self.sync_sm()

        # create default analyzer template
        self._create_default_template('analyzer-template', 'analyzer',
                                      flavor='m1.medium',
                                      image_name='analyzer')
        # create default NAT template
        self._create_default_template('nat-template', 'firewall',
                                      svc_mode='in-network-nat',
                                      image_name='analyzer',
                                      flavor='m1.medium')
        # create default netns SNAT template
        self._create_default_template('netns-snat-template', 'source-nat',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        # create default loadbalancer template
        self._create_default_template('haproxy-loadbalancer-template',
                                      'loadbalancer',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        self._create_default_template('docker-template', 'firewall',
                                      svc_mode='transparent',
                                      image_name="ubuntu",
                                      hypervisor_type='vrouter-instance',
                                      vrouter_instance_type='docker',
                                      instance_data={
                                          "command": "/bin/bash"
                                      })

        # upgrade handling
        self.upgrade()

        # check services
        self.launch_services()

        self._db_resync_done.set()

    def upgrade(self):
        for si in ServiceInstanceSM.values():
            st = ServiceTemplateSM.get(si.service_template)
            if not st:
                continue

            vm_id_list = list(si.virtual_machines)
            for vm_id in vm_id_list:
                vm = VirtualMachineSM.get(vm_id)
                if vm.virtualization_type:
                    continue

                try:
                    nova_vm = self._nova_client.oper('servers', 'get',
                        si.proj_name, id=vm_id)
                except nc_exc.NotFound:
                    nova_vm = None

                if nova_vm:
                    vm_name = nova_vm.name
                    vm.proj_fq_name = nova_vm.name.split('__')[0:2]
                else:
                    vm_name = vm.name

                if not vm_name.split('__')[-1].isdigit():
                    continue

                vm.virtualization_type = st.virtualization_type
                self._delete_service_instance(vm)

    def launch_services(self):
        for si in ServiceInstanceSM.values():
            self._create_service_instance(si)

    def sync_sm(self):
        # Read and Sync all DBase
        for cls in DBBaseSM.get_obj_type_map().values():
            for obj in cls.list_obj():
                cls.locate(obj['uuid'], obj)

        # Link SI and VM
        for vm in VirtualMachineSM.values():
            if vm.service_instance:
                continue
            for vmi_id in vm.virtual_machine_interfaces:
                vmi = VirtualMachineInterfaceSM.get(vmi_id)
                if not vmi:
                    continue
                self.port_delete_or_si_link(vm, vmi)

        # Load the loadbalancer driver
        self.loadbalancer_agent.load_drivers()

        # Invoke the loadbalancer pools
        for lb_pool in LoadbalancerPoolSM.values():
            lb_pool.add()

        # Audit the lb pools
        self.loadbalancer_agent.audit_lb_pools()

        # Audit the SNAT instances
        self.snat_agent.audit_snat_instances()
    # end sync_sm

    # create service template
    def _create_default_template(self, st_name, svc_type, svc_mode=None,
                                 hypervisor_type='virtual-machine',
                                 image_name=None, flavor=None, scaling=False,
                                 vrouter_instance_type=None,
                                 instance_data=None):
        domain_name = 'default-domain'
        domain_fq_name = [domain_name]
        st_fq_name = [domain_name, st_name]
        self.logger.log_info("Creating %s %s hypervisor %s" %
                             (domain_name, st_name, hypervisor_type))

        domain_obj = None
        for domain in DomainSM.values():
            if domain.fq_name == domain_fq_name:
                domain_obj = Domain()
                domain_obj.uuid = domain.uuid
                domain_obj.fq_name = domain_fq_name
                break
        if not domain_obj:
            self.logger.log_error("%s domain not found" % (domain_name))
            return

        for st in ServiceTemplateSM.values():
            if st.fq_name == st_fq_name:
                self.logger.log_info("%s exists uuid %s" %
                                     (st.name, str(st.uuid)))
                return

        svc_properties = ServiceTemplateType()
        svc_properties.set_service_type(svc_type)
        svc_properties.set_service_mode(svc_mode)
        svc_properties.set_service_virtualization_type(hypervisor_type)
        svc_properties.set_image_name(image_name)
        svc_properties.set_flavor(flavor)
        svc_properties.set_ordered_interfaces(True)
        svc_properties.set_service_scaling(scaling)

        # set interface list
        if svc_type == 'analyzer':
            if_list = [['left', False]]
        elif hypervisor_type == 'network-namespace':
            if_list = [['right', True], ['left', True]]
        else:
            if_list = [
                ['management', False], ['left', False], ['right', False]]

        for itf in if_list:
            if_type = ServiceTemplateInterfaceType(shared_ip=itf[1])
            if_type.set_service_interface_type(itf[0])
            svc_properties.add_interface_type(if_type)

        if vrouter_instance_type is not None:
            svc_properties.set_vrouter_instance_type(vrouter_instance_type)

        if instance_data is not None:
            svc_properties.set_instance_data(
                json.dumps(instance_data, separators=(',', ':')))

        st_obj = ServiceTemplate(name=st_name, domain_obj=domain)
        st_obj.set_service_template_properties(svc_properties)
        try:
            st_uuid = self._vnc_lib.service_template_create(st_obj)
        except Exception as e:
            self.logger.log_error("%s create failed with error %s" %
                                  (st_name, str(e)))
            return

        # Create the service template in local db
        ServiceTemplateSM.locate(st_uuid)

        self.logger.log_info("%s created with uuid %s" %
                             (st_name, str(st_uuid)))
    #_create_default_analyzer_template

    def port_delete_or_si_link(self, vm, vmi):
        if vmi.port_tuple:
            return
        if (vmi.service_instance and vmi.virtual_machine == None):
            self.vm_manager.cleanup_svc_vm_ports([vmi.uuid])
            return

        if not vm or vm.service_instance:
            return
        if not vmi.if_type:
            return

        si_fq_name = vmi.name.split('__')[0:3]
        index = int(vmi.name.split('__')[3]) - 1
        for si in ServiceInstanceSM.values():
            if si.fq_name != si_fq_name:
                continue
            st = ServiceTemplateSM.get(si.service_template)
            self.vm_manager.link_si_to_vm(si, st, index, vm.uuid)
            return

    def _create_service_instance(self, si):
        if si.state == 'active':
            return
        st = ServiceTemplateSM.get(si.service_template)
        if not st:
            self.logger.log_error("template not found for %s" %
                                  ((':').join(si.fq_name)))
            return
        if st.params and st.params.get('version', 1) == 2:
            return

        self.logger.log_info("Creating SI %s (%s)" %
                             ((':').join(si.fq_name), st.virtualization_type))
        try:
            if st.virtualization_type == 'virtual-machine':
                self.vm_manager.create_service(st, si)
            elif st.virtualization_type == 'network-namespace':
                self.netns_manager.create_service(st, si)
            elif st.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.create_service(st, si)
            elif st.virtualization_type == 'physical-device':
                self.ps_manager.create_service(st, si)
            else:
                self.logger.log_error("Unknown virt type: %s" %
                                      st.virtualization_type)
        except Exception:
            cgitb_error_log(self)
        si.launch_count += 1
        self.logger.log_info("SI %s creation success" % (':').join(si.fq_name))

    def _delete_service_instance(self, vm):
        self.logger.log_info("Deleting VM %s %s" %
            ((':').join(vm.fq_name), vm.uuid))

        try:
            if vm.virtualization_type == svc_info.get_vm_instance_type():
                self.vm_manager.delete_service(vm)
            elif vm.virtualization_type == svc_info.get_netns_instance_type():
                self.netns_manager.delete_service(vm)
            elif vm.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.delete_service(vm)
            elif vm.virtualization_type == 'physical-device':
                self.ps_manager.delete_service(vm)
        except Exception:
            cgitb_error_log(self)

        # generate UVE
        si_fq_name = vm.display_name.split('__')[:-2]
        si_fq_str = (':').join(si_fq_name)
        self.logger.uve_svc_instance(si_fq_str, status='DELETE',
                                     vms=[{'uuid': vm.uuid}])
        return True

    def _relaunch_service_instance(self, si):
        si.state = 'relaunch'
        self._create_service_instance(si)

    def _check_service_running(self, si):
        st = ServiceTemplateSM.get(si.service_template)
        if st.params and st.params.get('version', 1) == 2:
            return
        if st.virtualization_type == 'virtual-machine':
            status = self.vm_manager.check_service(si)
        elif st.virtualization_type == 'network-namespace':
            status = self.netns_manager.check_service(si)
        elif st.virtualization_type == 'vrouter-instance':
            status = self.vrouter_manager.check_service(si)
        elif st.virtualization_type == 'physical-device':
            status = self.ps_manager.check_service(si)
        return status

    def _delete_interface_route_table(self, irt_uuid):
        try:
            self._vnc_lib.interface_route_table_delete(id=irt_uuid)
            InterfaceRouteTableSM.delete(irt_uuid)
        except (NoIdError, RefsExistError):
            return

    def _delete_shared_vn(self, vn_uuid):
        try:
            self.logger.log_info("Deleting vn %s" % (vn_uuid))
            self._vnc_lib.virtual_network_delete(id=vn_uuid)
            VirtualNetworkSM.delete(vn_uuid)
        except (NoIdError, RefsExistError):
            pass

    @staticmethod
    def reset():
        for cls in DBBaseSM.get_obj_type_map().values():
            cls.reset()
class SvcMonitor(object):
    def __init__(self, args=None):
        self._args = args

        # initialize discovery client
        self._disc = None
        if self._args.disc_server_ip and self._args.disc_server_port:
            self._disc = client.DiscoveryClient(
                self._args.disc_server_ip, self._args.disc_server_port,
                ModuleNames[Module.SVC_MONITOR])
        # initialize logger
        self.logger = ServiceMonitorLogger(self._disc, args)

        # rotating log file for catchall errors
        self._err_file = self._args.trace_file
        self._svc_err_logger = logging.getLogger('SvcErrLogger')
        self._svc_err_logger.setLevel(logging.ERROR)
        try:
            with open(self._err_file, 'a'):
                handler = logging.handlers.RotatingFileHandler(self._err_file,
                                                               maxBytes=64 *
                                                               1024,
                                                               backupCount=2)
                self._svc_err_logger.addHandler(handler)
        except IOError:
            self.logger.warning("Failed to open trace file %s" %
                                self._err_file)

        # init cassandra
        self._cassandra = ServiceMonitorDB(self._args, self.logger)
        DBBaseSM.init(self, self.logger, self._cassandra)

        # init rabbit connection
        self.rabbit = RabbitConnection(self.logger, args)
        self.rabbit._connect_rabbit()

    def post_init(self, vnc_lib, args=None):
        # api server
        self._vnc_lib = vnc_lib

        try:
            self._nova_client = importutils.import_object(
                'svc_monitor.nova_client.ServiceMonitorNovaClient', self._args,
                self.logger)
        except Exception as e:
            self._nova_client = None

        # agent manager
        self._agent_manager = AgentManager()

        # load vrouter scheduler
        self.vrouter_scheduler = importutils.import_object(
            self._args.si_netns_scheduler_driver, self._vnc_lib,
            self._nova_client, self._disc, self.logger, self._args)

        # load virtual machine instance manager
        self.vm_manager = importutils.import_object(
            'svc_monitor.virtual_machine_manager.VirtualMachineManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load network namespace instance manager
        self.netns_manager = importutils.import_object(
            'svc_monitor.instance_manager.NetworkNamespaceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load a vrouter instance manager
        self.vrouter_manager = importutils.import_object(
            'svc_monitor.vrouter_instance_manager.VRouterInstanceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load PNF instance manager
        self.ps_manager = importutils.import_object(
            'svc_monitor.physical_service_manager.PhysicalServiceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load a loadbalancer agent
        self.loadbalancer_agent = LoadbalancerAgent(self, self._vnc_lib,
                                                    self._cassandra,
                                                    self._args)
        self._agent_manager.register_agent(self.loadbalancer_agent)

        # load a snat agent
        self.snat_agent = SNATAgent(self, self._vnc_lib, self._cassandra,
                                    self._args)
        self._agent_manager.register_agent(self.snat_agent)

        # load port tuple agent
        self.port_tuple_agent = PortTupleAgent(self, self._vnc_lib,
                                               self._cassandra, self._args,
                                               self.logger)
        self._agent_manager.register_agent(self.port_tuple_agent)

        # Read the cassandra and populate the entry in ServiceMonitor DB
        self.sync_sm()

        # create default analyzer template
        self._create_default_template('analyzer-template',
                                      'analyzer',
                                      flavor='m1.medium',
                                      image_name='analyzer')
        # create default NAT template
        self._create_default_template('nat-template',
                                      'firewall',
                                      svc_mode='in-network-nat',
                                      image_name='analyzer',
                                      flavor='m1.medium')
        # create default netns SNAT template
        self._create_default_template('netns-snat-template',
                                      'source-nat',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        # create default loadbalancer template
        self._create_default_template('haproxy-loadbalancer-template',
                                      'loadbalancer',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        self._create_default_template('docker-template',
                                      'firewall',
                                      svc_mode='transparent',
                                      image_name="ubuntu",
                                      hypervisor_type='vrouter-instance',
                                      vrouter_instance_type='docker',
                                      instance_data={"command": "/bin/bash"})

        # upgrade handling
        self.upgrade()

        # check services
        self.vrouter_scheduler.vrouters_running()
        self.launch_services()

        self.rabbit._db_resync_done.set()

    def _upgrade_instance_ip(self, vm):
        for vmi_id in vm.virtual_machine_interfaces:
            vmi = VirtualMachineInterfaceSM.get(vmi_id)
            if not vmi:
                continue

            for iip_id in vmi.instance_ips:
                iip = InstanceIpSM.get(iip_id)
                if not iip or iip.service_instance_ip:
                    continue
                iip_obj = InstanceIp()
                iip_obj.name = iip.name
                iip_obj.uuid = iip.uuid
                iip_obj.set_service_instance_ip(True)
                try:
                    self._vnc_lib.instance_ip_update(iip_obj)
                except NoIdError:
                    self.logger.error(
                        "upgrade instance ip to service ip failed %s" %
                        (iip.name))
                    continue

    def _upgrade_auto_policy(self, si, st):
        if st.name != 'netns-snat-template':
            return
        if not si.params['auto_policy']:
            return

        si_obj = ServiceInstance()
        si_obj.uuid = si.uuid
        si_obj.fq_name = si.fq_name
        si_props = ServiceInstanceType(**si.params)
        si_props.set_auto_policy(False)
        si_obj.set_service_instance_properties(si_props)
        try:
            self._vnc_lib.service_instance_update(si_obj)
            self.logger.notice("snat policy upgraded for %s" % (si.name))
        except NoIdError:
            self.logger.error("snat policy upgrade failed for %s" % (si.name))
            return

    def upgrade(self):
        for lr in LogicalRouterSM.values():
            self.snat_agent.upgrade(lr)

        for si in ServiceInstanceSM.values():
            st = ServiceTemplateSM.get(si.service_template)
            if not st:
                continue

            self._upgrade_auto_policy(si, st)

            vm_id_list = list(si.virtual_machines)
            for vm_id in vm_id_list:
                vm = VirtualMachineSM.get(vm_id)
                self._upgrade_instance_ip(vm)
                if vm.virtualization_type:
                    continue

                try:
                    nova_vm = self._nova_client.oper('servers',
                                                     'get',
                                                     si.proj_name,
                                                     id=vm_id)
                except nc_exc.NotFound:
                    nova_vm = None

                if nova_vm:
                    vm_name = nova_vm.name
                    vm.proj_fq_name = nova_vm.name.split('__')[0:2]
                else:
                    vm_name = vm.name

                if not vm_name.split('__')[-1].isdigit():
                    continue

                vm.virtualization_type = st.virtualization_type
                self.delete_service_instance(vm)

    def launch_services(self):
        for si in ServiceInstanceSM.values():
            self.create_service_instance(si)

    def sync_sm(self):
        # Read and Sync all DBase
        for cls in DBBaseSM.get_obj_type_map().values():
            for obj in cls.list_obj():
                cls.locate(obj['uuid'], obj)

        # Link SI and VM
        for vm in VirtualMachineSM.values():
            if vm.service_instance:
                continue
            for vmi_id in vm.virtual_machine_interfaces:
                vmi = VirtualMachineInterfaceSM.get(vmi_id)
                if not vmi:
                    continue
                self.port_delete_or_si_link(vm, vmi)

        # invoke port tuple handling
        try:
            self.port_tuple_agent.update_port_tuples()
        except Exception:
            cgitb_error_log(self)

        # Load the loadbalancer driver
        self.loadbalancer_agent.load_drivers()

        # Invoke the loadbalancers
        for lb in LoadbalancerSM.values():
            lb.add()

        # Invoke the loadbalancer listeners
        for lb_listener in LoadbalancerListenerSM.values():
            lb_listener.add()

        # Invoke the loadbalancer pools
        for lb_pool in LoadbalancerPoolSM.values():
            lb_pool.add()

        # Audit the lb pools
        self.loadbalancer_agent.audit_lb_pools()

        # Audit the SNAT instances
        self.snat_agent.audit_snat_instances()

    # end sync_sm

    # create service template
    def _create_default_template(self,
                                 st_name,
                                 svc_type,
                                 svc_mode=None,
                                 hypervisor_type='virtual-machine',
                                 image_name=None,
                                 flavor=None,
                                 scaling=False,
                                 vrouter_instance_type=None,
                                 instance_data=None):
        domain_name = 'default-domain'
        domain_fq_name = [domain_name]
        st_fq_name = [domain_name, st_name]
        self.logger.info("Creating %s %s hypervisor %s" %
                         (domain_name, st_name, hypervisor_type))

        domain_obj = None
        for domain in DomainSM.values():
            if domain.fq_name == domain_fq_name:
                domain_obj = Domain()
                domain_obj.uuid = domain.uuid
                domain_obj.fq_name = domain_fq_name
                break
        if not domain_obj:
            self.logger.error("%s domain not found" % (domain_name))
            return

        for st in ServiceTemplateSM.values():
            if st.fq_name == st_fq_name:
                self.logger.info("%s exists uuid %s" % (st.name, str(st.uuid)))
                return

        svc_properties = ServiceTemplateType()
        svc_properties.set_service_type(svc_type)
        svc_properties.set_service_mode(svc_mode)
        svc_properties.set_service_virtualization_type(hypervisor_type)
        svc_properties.set_image_name(image_name)
        svc_properties.set_flavor(flavor)
        svc_properties.set_ordered_interfaces(True)
        svc_properties.set_service_scaling(scaling)

        # set interface list
        if svc_type == 'analyzer':
            if_list = [['left', False]]
        elif hypervisor_type == 'network-namespace':
            if_list = [['right', True], ['left', True]]
        else:
            if_list = [['management', False], ['left', False],
                       ['right', False]]

        for itf in if_list:
            if_type = ServiceTemplateInterfaceType(shared_ip=itf[1])
            if_type.set_service_interface_type(itf[0])
            svc_properties.add_interface_type(if_type)

        if vrouter_instance_type is not None:
            svc_properties.set_vrouter_instance_type(vrouter_instance_type)

        if instance_data is not None:
            svc_properties.set_instance_data(
                json.dumps(instance_data, separators=(',', ':')))

        st_obj = ServiceTemplate(name=st_name, domain_obj=domain)
        st_obj.set_service_template_properties(svc_properties)
        try:
            st_uuid = self._vnc_lib.service_template_create(st_obj)
        except Exception as e:
            self.logger.error("%s create failed with error %s" %
                              (st_name, str(e)))
            return

        # Create the service template in local db
        ServiceTemplateSM.locate(st_uuid)

        self.logger.info("%s created with uuid %s" % (st_name, str(st_uuid)))

    #_create_default_analyzer_template

    def port_delete_or_si_link(self, vm, vmi):
        if vmi.port_tuple:
            return
        if (vmi.service_instance and vmi.virtual_machine == None):
            self.vm_manager.cleanup_svc_vm_ports([vmi.uuid])
            return

        if not vm or vm.service_instance:
            return
        if not vmi.if_type:
            return

        if len(vmi.name.split('__')) < 4:
            return
        si_fq_name = vmi.name.split('__')[0:3]
        index = int(vmi.name.split('__')[3]) - 1
        for si in ServiceInstanceSM.values():
            if si.fq_name != si_fq_name:
                continue
            st = ServiceTemplateSM.get(si.service_template)
            self.vm_manager.link_si_to_vm(si, st, index, vm.uuid)
            return

    def create_service_instance(self, si):
        if si.state == 'active':
            return
        st = ServiceTemplateSM.get(si.service_template)
        if not st:
            self.logger.error("template not found for %s" %
                              ((':').join(si.fq_name)))
            return
        if st.params and st.params.get('version', 1) == 2:
            return

        self.logger.info("Creating SI %s (%s)" %
                         ((':').join(si.fq_name), st.virtualization_type))
        try:
            if st.virtualization_type == 'virtual-machine':
                self.vm_manager.create_service(st, si)
            elif st.virtualization_type == 'network-namespace':
                self.netns_manager.create_service(st, si)
            elif st.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.create_service(st, si)
            elif st.virtualization_type == 'physical-device':
                self.ps_manager.create_service(st, si)
            else:
                self.logger.error("Unknown virt type: %s" %
                                  st.virtualization_type)
        except Exception:
            cgitb_error_log(self)
        si.launch_count += 1
        self.logger.info("SI %s creation success" % (':').join(si.fq_name))

    def delete_service_instance(self, vm):
        self.logger.info("Deleting VM %s %s for SI %s" %
                         ((':').join(vm.fq_name), vm.uuid, vm.service_id))

        try:
            if vm.virtualization_type == svc_info.get_vm_instance_type():
                self.vm_manager.delete_service(vm)
            elif vm.virtualization_type == svc_info.get_netns_instance_type():
                self.netns_manager.delete_service(vm)
            elif vm.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.delete_service(vm)
            elif vm.virtualization_type == 'physical-device':
                self.ps_manager.delete_service(vm)
            self.logger.info("Deleted VM %s %s for SI %s" %
                             ((':').join(vm.fq_name), vm.uuid, vm.service_id))
        except Exception:
            cgitb_error_log(self)

        # generate UVE
        si_fq_name = vm.display_name.split('__')[:-2]
        si_fq_str = (':').join(si_fq_name)
        self.logger.uve_svc_instance(si_fq_str,
                                     status='DELETE',
                                     vms=[{
                                         'uuid': vm.uuid
                                     }])
        return True

    def _relaunch_service_instance(self, si):
        si.state = 'relaunch'
        self.create_service_instance(si)

    def _check_service_running(self, si):
        st = ServiceTemplateSM.get(si.service_template)
        if st.params and st.params.get('version', 1) == 2:
            return
        if st.virtualization_type == 'virtual-machine':
            status = self.vm_manager.check_service(si)
        elif st.virtualization_type == 'network-namespace':
            status = self.netns_manager.check_service(si)
        elif st.virtualization_type == 'vrouter-instance':
            status = self.vrouter_manager.check_service(si)
        elif st.virtualization_type == 'physical-device':
            status = self.ps_manager.check_service(si)
        return status

    def delete_interface_route_table(self, irt_uuid):
        try:
            self._vnc_lib.interface_route_table_delete(id=irt_uuid)
            InterfaceRouteTableSM.delete(irt_uuid)
        except (NoIdError, RefsExistError):
            return

    def _delete_shared_vn(self, vn_uuid):
        try:
            self.logger.info("Deleting vn %s" % (vn_uuid))
            self._vnc_lib.virtual_network_delete(id=vn_uuid)
            VirtualNetworkSM.delete(vn_uuid)
        except (NoIdError, RefsExistError):
            pass

    @staticmethod
    def reset():
        for cls in DBBaseSM.get_obj_type_map().values():
            cls.reset()
class SvcMonitor(object):

    def __init__(self, sm_logger=None, args=None):
        self._args = args
        # initialize logger
        if sm_logger is not None:
            self.logger = sm_logger
        else:
            # Initialize logger
            self.logger = ServiceMonitorLogger(args)

        # init object_db
        self._object_db = ServiceMonitorDB(self._args, self.logger)
        DBBaseSM.init(self, self.logger, self._object_db)

        # init rabbit connection
        rabbitmq_cfg = get_rabbitmq_cfg(args)
        self.rabbit = VncAmqpHandle(self.logger._sandesh, self.logger,
                DBBaseSM, REACTION_MAP, 'svc_monitor', rabbitmq_cfg,
                self._args.trace_file)
        self.rabbit.establish()

    def post_init(self, vnc_lib, args=None):
        # api server
        self._vnc_lib = vnc_lib

        try:
            self._nova_client = importutils.import_object(
                'svc_monitor.nova_client.ServiceMonitorNovaClient',
                self._args, self.logger)
        except Exception as e:
            self._nova_client = None

        # agent manager
        self._agent_manager = AgentManager()

        # load vrouter scheduler
        self.vrouter_scheduler = importutils.import_object(
            self._args.si_netns_scheduler_driver,
            self._vnc_lib, self._nova_client,
            None, self.logger, self._args)

        # load virtual machine instance manager
        self.vm_manager = importutils.import_object(
            'svc_monitor.virtual_machine_manager.VirtualMachineManager',
            self._vnc_lib, self._object_db, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load network namespace instance manager
        self.netns_manager = importutils.import_object(
            'svc_monitor.instance_manager.NetworkNamespaceManager',
            self._vnc_lib, self._object_db, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load a vrouter instance manager
        self.vrouter_manager = importutils.import_object(
            'svc_monitor.vrouter_instance_manager.VRouterInstanceManager',
            self._vnc_lib, self._object_db, self.logger,
            self.vrouter_scheduler, self._nova_client,
            self._agent_manager, self._args)

        # load PNF instance manager
        self.ps_manager = importutils.import_object(
            'svc_monitor.physical_service_manager.PhysicalServiceManager',
            self._vnc_lib, self._object_db, self.logger,
            self.vrouter_scheduler, self._nova_client,
            self._agent_manager, self._args)

        # load a loadbalancer agent
        self.loadbalancer_agent = LoadbalancerAgent(
            self, self._vnc_lib,
            self._object_db, self._args)
        self._agent_manager.register_agent(self.loadbalancer_agent)

        # load a snat agent
        self.snat_agent = SNATAgent(self, self._vnc_lib,
                                    self._object_db, self._args,
                                    ServiceMonitorModuleLogger(self.logger))
        self._agent_manager.register_agent(self.snat_agent)

        # load port tuple agent
        self.port_tuple_agent = PortTupleAgent(self, self._vnc_lib,
            self._object_db, self._args, ServiceMonitorModuleLogger(self.logger))
        self._agent_manager.register_agent(self.port_tuple_agent)

        # Read the object_db and populate the entry in ServiceMonitor DB
        self.sync_sm()

        # create default analyzer template
        self._create_default_template('analyzer-template', 'analyzer',
                                      flavor='m1.medium',
                                      image_name='analyzer')
        # create default NAT template
        self._create_default_template('nat-template', 'firewall',
                                      svc_mode='in-network-nat',
                                      image_name='analyzer',
                                      flavor='m1.medium')
        # create default netns SNAT template
        self._create_default_template('netns-snat-template', 'source-nat',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        # create default loadbalancer template
        self._create_default_template('haproxy-loadbalancer-template',
                                      'loadbalancer',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        self._create_default_template('docker-template', 'firewall',
                                      svc_mode='transparent',
                                      image_name="ubuntu",
                                      hypervisor_type='vrouter-instance',
                                      vrouter_instance_type='docker',
                                      instance_data={
                                          "command": "/bin/bash"
                                      })

        # upgrade handling
        self.upgrade()

        # check services
        self.vrouter_scheduler.vrouters_running()
        self.launch_services()

        self.rabbit._db_resync_done.set()

    def _upgrade_instance_ip(self, vm):
        for vmi_id in vm.virtual_machine_interfaces:
            vmi = VirtualMachineInterfaceSM.get(vmi_id)
            if not vmi:
                continue

            for iip_id in vmi.instance_ips:
                iip = InstanceIpSM.get(iip_id)
                if not iip or iip.service_instance_ip:
                    continue
                iip_obj = InstanceIp()
                iip_obj.name = iip.name
                iip_obj.uuid = iip.uuid
                iip_obj.set_service_instance_ip(True)
                try:
                    self._vnc_lib.instance_ip_update(iip_obj)
                except NoIdError:
                    self.logger.error("upgrade instance ip to service ip failed %s" % (iip.name))
                    continue

    def _upgrade_auto_policy(self, si, st):
        if st.name != 'netns-snat-template':
            return
        if not si.params['auto_policy']:
            return

        si_obj = ServiceInstance()
        si_obj.uuid = si.uuid
        si_obj.fq_name = si.fq_name
        si_props = ServiceInstanceType(**si.params)
        si_props.set_auto_policy(False)
        si_obj.set_service_instance_properties(si_props)
        try:
            self._vnc_lib.service_instance_update(si_obj)
            self.logger.notice("snat policy upgraded for %s" % (si.name))
        except NoIdError:
            self.logger.error("snat policy upgrade failed for %s" % (si.name))
            return

    def upgrade(self):
        for lr in LogicalRouterSM.values():
            self.snat_agent.upgrade(lr)

        for si in ServiceInstanceSM.values():
            st = ServiceTemplateSM.get(si.service_template)
            if not st:
                continue

            self._upgrade_auto_policy(si, st)

            vm_id_list = list(si.virtual_machines)
            for vm_id in vm_id_list:
                vm = VirtualMachineSM.get(vm_id)
                self._upgrade_instance_ip(vm)
                if vm.virtualization_type:
                    continue

                try:
                    nova_vm = self._nova_client.oper('servers', 'get',
                        si.proj_name, id=vm_id)
                except nc_exc.NotFound:
                    nova_vm = None

                if nova_vm:
                    vm_name = nova_vm.name
                    vm.proj_fq_name = nova_vm.name.split('__')[0:2]
                else:
                    vm_name = vm.name

                if not vm_name.split('__')[-1].isdigit():
                    continue

                vm.virtualization_type = st.virtualization_type
                self.delete_service_instance(vm)

    def launch_services(self):
        for si in ServiceInstanceSM.values():
            self.create_service_instance(si)

    def sync_sm(self):
        # Read and Sync all DBase
        for cls in DBBaseSM.get_obj_type_map().values():
            for obj in cls.list_obj():
                cls.locate(obj['uuid'], obj)

        # Link SI and VM
        for vm in VirtualMachineSM.values():
            if vm.service_instance:
                continue
            for vmi_id in vm.virtual_machine_interfaces:
                vmi = VirtualMachineInterfaceSM.get(vmi_id)
                if not vmi:
                    continue
                self.port_delete_or_si_link(vm, vmi)

        # invoke port tuple handling
        try:
            self.port_tuple_agent.update_port_tuples()
        except Exception:
            cgitb_error_log(self)

        # Load the loadbalancer driver
        self.loadbalancer_agent.load_drivers()

        # Invoke the health monitors
        for hm in HealthMonitorSM.values():
            hm.sync()

        # Invoke the loadbalancers
        for lb in LoadbalancerSM.values():
            lb.sync()

        # Invoke the loadbalancer listeners
        for lb_listener in LoadbalancerListenerSM.values():
            lb_listener.sync()

        # Invoke the loadbalancer pools
        for lb_pool in LoadbalancerPoolSM.values():
            lb_pool.sync()

        # Audit the lb pools
        self.loadbalancer_agent.audit_lb_pools()

        # Audit the SNAT instances
        self.snat_agent.audit_snat_instances()
    # end sync_sm

    # create service template
    def _create_default_template(self, st_name, svc_type, svc_mode=None,
                                 hypervisor_type='virtual-machine',
                                 image_name=None, flavor=None, scaling=False,
                                 vrouter_instance_type=None,
                                 instance_data=None):
        domain_name = 'default-domain'
        domain_fq_name = [domain_name]
        st_fq_name = [domain_name, st_name]
        self.logger.info("Creating %s %s hypervisor %s" %
                             (domain_name, st_name, hypervisor_type))

        domain_obj = None
        for domain in DomainSM.values():
            if domain.fq_name == domain_fq_name:
                domain_obj = Domain()
                domain_obj.uuid = domain.uuid
                domain_obj.fq_name = domain_fq_name
                break
        if not domain_obj:
            self.logger.error("%s domain not found" % (domain_name))
            return

        for st in ServiceTemplateSM.values():
            if st.fq_name == st_fq_name:
                self.logger.info("%s exists uuid %s" %
                                     (st.name, str(st.uuid)))
                return

        svc_properties = ServiceTemplateType()
        svc_properties.set_service_type(svc_type)
        svc_properties.set_service_mode(svc_mode)
        svc_properties.set_service_virtualization_type(hypervisor_type)
        svc_properties.set_image_name(image_name)
        svc_properties.set_flavor(flavor)
        svc_properties.set_ordered_interfaces(True)
        svc_properties.set_service_scaling(scaling)

        # set interface list
        if svc_type == 'analyzer':
            if_list = [['left', False]]
        elif hypervisor_type == 'network-namespace':
            if_list = [['right', True], ['left', True]]
        else:
            if_list = [
                ['management', False], ['left', False], ['right', False]]

        for itf in if_list:
            if_type = ServiceTemplateInterfaceType(shared_ip=itf[1])
            if_type.set_service_interface_type(itf[0])
            svc_properties.add_interface_type(if_type)

        if vrouter_instance_type is not None:
            svc_properties.set_vrouter_instance_type(vrouter_instance_type)

        if instance_data is not None:
            svc_properties.set_instance_data(
                json.dumps(instance_data, separators=(',', ':')))

        st_obj = ServiceTemplate(name=st_name, domain_obj=domain)
        st_obj.set_service_template_properties(svc_properties)
        try:
            st_uuid = self._vnc_lib.service_template_create(st_obj)
        except Exception as e:
            self.logger.error("%s create failed with error %s" %
                                  (st_name, str(e)))
            return

        # Create the service template in local db
        ServiceTemplateSM.locate(st_uuid)

        self.logger.info("%s created with uuid %s" %
                             (st_name, str(st_uuid)))
    #_create_default_analyzer_template

    def port_delete_or_si_link(self, vm, vmi):
        if vmi.port_tuples:
            return
        if (vmi.service_instances and vmi.virtual_machine == None):
            self.vm_manager.cleanup_svc_vm_ports([vmi.uuid])
            return

        if not vm or vm.service_instance:
            return
        if not vmi.if_type:
            return

        if len(vmi.name.split('__')) < 4:
            return
        si_fq_name = vmi.name.split('__')[0:3]
        index = int(vmi.name.split('__')[3]) - 1
        for si in ServiceInstanceSM.values():
            if si.fq_name != si_fq_name:
                continue
            st = ServiceTemplateSM.get(si.service_template)
            self.vm_manager.link_si_to_vm(si, st, index, vm.uuid)
            return

    def create_service_instance(self, si):
        if si.state == 'active':
            return
        st = ServiceTemplateSM.get(si.service_template)
        if not st:
            self.logger.error("template not found for %s" %
                                  ((':').join(si.fq_name)))
            return
        if st.params and st.params.get('version', 1) == 2:
            return

        self.logger.info("Creating SI %s (%s)" %
                             ((':').join(si.fq_name), st.virtualization_type))
        try:
            if st.virtualization_type == 'virtual-machine':
                self.vm_manager.create_service(st, si)
            elif st.virtualization_type == 'network-namespace':
                self.netns_manager.create_service(st, si)
            elif st.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.create_service(st, si)
            elif st.virtualization_type == 'physical-device':
                self.ps_manager.create_service(st, si)
            else:
                self.logger.error("Unknown virt type: %s" %
                                      st.virtualization_type)
        except Exception:
            cgitb_error_log(self)
        si.launch_count += 1
        self.logger.info("SI %s creation success" % (':').join(si.fq_name))

    def delete_service_instance(self, vm):
        self.logger.info("Deleting VM %s %s for SI %s" %
            ((':').join(vm.fq_name), vm.uuid, vm.service_id))

        try:
            if vm.virtualization_type == svc_info.get_vm_instance_type():
                self.vm_manager.delete_service(vm)
            elif vm.virtualization_type == svc_info.get_netns_instance_type():
                self.netns_manager.delete_service(vm)
            elif vm.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.delete_service(vm)
            elif vm.virtualization_type == 'physical-device':
                self.ps_manager.delete_service(vm)
            self.logger.info("Deleted VM %s %s for SI %s" %
                ((':').join(vm.fq_name), vm.uuid, vm.service_id))
        except Exception:
            cgitb_error_log(self)

        # generate UVE
        si_fq_name = vm.display_name.split('__')[:-2]
        si_fq_str = (':').join(si_fq_name)
        self.logger.uve_svc_instance(si_fq_str, status='DELETE',
                                     vms=[{'uuid': vm.uuid}])
        return True

    def _relaunch_service_instance(self, si):
        si.state = 'relaunch'
        self.create_service_instance(si)

    def _check_service_running(self, si):
        st = ServiceTemplateSM.get(si.service_template)
        if st.params and st.params.get('version', 1) == 2:
            return
        if st.virtualization_type == 'virtual-machine':
            status = self.vm_manager.check_service(si)
        elif st.virtualization_type == 'network-namespace':
            status = self.netns_manager.check_service(si)
        elif st.virtualization_type == 'vrouter-instance':
            status = self.vrouter_manager.check_service(si)
        elif st.virtualization_type == 'physical-device':
            status = self.ps_manager.check_service(si)
        return status

    def delete_interface_route_table(self, irt_uuid):
        try:
            self._vnc_lib.interface_route_table_delete(id=irt_uuid)
            InterfaceRouteTableSM.delete(irt_uuid)
        except (NoIdError, RefsExistError):
            return

    def _delete_shared_vn(self, vn_uuid):
        try:
            self.logger.info("Deleting vn %s" % (vn_uuid))
            self._vnc_lib.virtual_network_delete(id=vn_uuid)
            VirtualNetworkSM.delete(vn_uuid)
        except (NoIdError, RefsExistError):
            pass

    @staticmethod
    def reset():
        for cls in DBBaseSM.get_obj_type_map().values():
            cls.reset()

    def sighup_handler(self):
        if self._conf_file:
            config = ConfigParser.SafeConfigParser()
            config.read(self._conf_file)
            if 'DEFAULTS' in config.sections():
                try:
                    collectors = config.get('DEFAULTS', 'collectors')
                    if type(collectors) is str:
                        collectors = collectors.split()
                        new_chksum = hashlib.md5("".join(collectors)).hexdigest()
                        if new_chksum != self._chksum:
                            self._chksum = new_chksum
                            config.random_collectors = random.sample(collectors, len(collectors))
                        # Reconnect to achieve load-balance irrespective of list
                        self.logger.sandesh_reconfig_collectors(config)
                except ConfigParser.NoOptionError as e:
                     pass
class SvcMonitor(object):

    """
    data + methods used/referred to by ssrc and arc greenlets
    """
    _REACTION_MAP = {
        "service_appliance_set": {
            'self': [],
            'service_appliance': []
        },
        "service_appliance": {
            'self': ['service_appliance_set'],
            'service_appliance_set': []
        },
        "loadbalancer_pool": {
            'self': [],
            'virtual_ip': [],
            'loadbalancer_member': [],
            'loadbalancer_healthmonitor': [],
        },
        "loadbalancer_member": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "virtual_ip": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "loadbalancer_healthmonitor": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "service_instance": {
            'self': ['virtual_machine'],
            'virtual_machine': []
        },
        "instance_ip": {
            'self': [],
        },
        "floating_ip": {
            'self': [],
        },
        "service_template": {
            'self': [],
        },
        "physical_router": {
            'self': [],
        },
        "physical_interface": {
            'self': [],
        },
        "logical_interface": {
            'self': [],
        },
        "logical_router": {
            'self': [],
        },
        "virtual_network": {
            'self': [],
        },
        "virtual_machine": {
            'self': ['virtual_machine_interface'],
            'service_instance': [],
            'virtual_machine_interface': [],
        },
        "virtual_machine_interface": {
            'self': ['interface_route_table', 'virtual_machine'],
            'interface_route_table': [],
            'virtual_machine': [],
        },
        "interface_route_table": {
            'self': [],
            'virtual_machine_interface': [],
        },
        "project": {
            'self': [],
        },
    }

    def __init__(self, args=None):
        self._args = args

        # create database and logger
        self.si_db = ServiceInstanceDB(args)

        # initialize discovery client
        self._disc = None
        if self._args.disc_server_ip and self._args.disc_server_port:
            self._disc = client.DiscoveryClient(self._args.disc_server_ip,
                                                self._args.disc_server_port,
                                                ModuleNames[Module.SVC_MONITOR])

        # initialize logger
        self.logger = ServiceMonitorLogger(self.si_db, self._disc, args)
        self.si_db.add_logger(self.logger)
        self.si_db.init_database()

        # rotating log file for catchall errors
        self._err_file = self._args.trace_file
        self._svc_err_logger = logging.getLogger('SvcErrLogger')
        self._svc_err_logger.setLevel(logging.ERROR)
        try:
            with open(self._err_file, 'a'):
                handler = logging.handlers.RotatingFileHandler(
                    self._err_file, maxBytes=64*1024, backupCount=2)
                self._svc_err_logger.addHandler(handler)
        except IOError:
            self.logger.log_warning("Failed to open trace file %s" %
                self._err_file)

        # Connect to Rabbit and Initialize cassandra connection
        self._connect_rabbit()

    def _connect_rabbit(self):
        rabbit_server = self._args.rabbit_server
        rabbit_port = self._args.rabbit_port
        rabbit_user = self._args.rabbit_user
        rabbit_password = self._args.rabbit_password
        rabbit_vhost = self._args.rabbit_vhost
        rabbit_ha_mode = self._args.rabbit_ha_mode

        self._db_resync_done = gevent.event.Event()

        q_name = 'svc_mon.%s' % (socket.gethostname())
        self._vnc_kombu = VncKombuClient(rabbit_server, rabbit_port,
                                         rabbit_user, rabbit_password,
                                         rabbit_vhost, rabbit_ha_mode,
                                         q_name, self._vnc_subscribe_callback,
                                         self.config_log)

        cass_server_list = self._args.cassandra_server_list
        reset_config = self._args.reset_config
        self._cassandra = VncCassandraClient(cass_server_list, reset_config,
                                             self._args.cluster_id, None,
                                             self.config_log)
        DBBase.init(self, self.logger, self._cassandra)
    # end _connect_rabbit

    def config_log(self, msg, level):
        self.logger.log(msg)

    def _vnc_subscribe_callback(self, oper_info):
        self._db_resync_done.wait()
        try:
            msg = "Notification Message: %s" % (pformat(oper_info))
            self.config_log(msg, level=SandeshLevel.SYS_DEBUG)
            obj_type = oper_info['type'].replace('-', '_')
            obj_class = DBBase._OBJ_TYPE_MAP.get(obj_type)
            if obj_class is None:
                return

            if oper_info['oper'] == 'CREATE' or oper_info['oper'] == 'UPDATE':
                dependency_tracker = DependencyTracker(DBBase._OBJ_TYPE_MAP,
                    self._REACTION_MAP)
                obj_id = oper_info['uuid']
                obj = obj_class.get(obj_id)
                if obj is not None:
                    dependency_tracker.evaluate(obj_type, obj)
                else:
                    obj = obj_class.locate(obj_id)
                obj.update()
                dependency_tracker.evaluate(obj_type, obj)
            elif oper_info['oper'] == 'DELETE':
                obj_id = oper_info['uuid']
                obj = obj_class.get(obj_id)
                if obj is None:
                    return
                dependency_tracker = DependencyTracker(DBBase._OBJ_TYPE_MAP,
                    self._REACTION_MAP)
                dependency_tracker.evaluate(obj_type, obj)
                obj_class.delete(obj_id)
            else:
                # unknown operation
                self.config_log('Unknown operation %s' % oper_info['oper'],
                                level=SandeshLevel.SYS_ERR)
                return

            if obj is None:
                self.config_log('Error while accessing %s uuid %s' % (
                                obj_type, obj_id))
                return


        except Exception:
            string_buf = cStringIO.StringIO()
            cgitb.Hook(file=string_buf, format="text").handle(sys.exc_info())
            self.config_log(string_buf.getvalue(), level=SandeshLevel.SYS_ERR)

        for sas_id in dependency_tracker.resources.get('service_appliance_set', []):
            sas_obj = ServiceApplianceSetSM.get(sas_id)
            if sas_obj is not None:
                sas_obj.add()

        for lb_pool_id in dependency_tracker.resources.get('loadbalancer_pool', []):
            lb_pool = LoadbalancerPoolSM.get(lb_pool_id)
            if lb_pool is not None:
                lb_pool.add()

        for si_id in dependency_tracker.resources.get('service_instance', []):
            si = ServiceInstanceSM.get(si_id)
            if si:
                self._create_service_instance(si)
            else:
                for vm_id in dependency_tracker.resources.get(
                        'virtual_machine', []):
                    vm = VirtualMachineSM.get(vm_id)
                    self._delete_service_instance(vm)

        for vn_id in dependency_tracker.resources.get('virtual_network', []):
            vn = VirtualNetworkSM.get(vn_id)
            if vn:
                for si_id in ServiceInstanceSM:
                    si = ServiceInstanceSM.get(si_id)
                    if (':').join(vn.fq_name) in si.params.values():
                        self._create_service_instance(si)

        for vmi_id in dependency_tracker.resources.get('virtual_machine_interface', []):
            vmi = VirtualMachineInterfaceSM.get(vmi_id)
            if vmi:
                for vm_id in dependency_tracker.resources.get(
                        'virtual_machine', []):
                    vm = VirtualMachineSM.get(vm_id)
                    if vm:
                        self.check_link_si_to_vm(vm, vmi)
            else:
                for irt_id in dependency_tracker.resources.get(
                        'interface_route_table', []):
                    self._delete_interface_route_table(irt_id)

        for fip_id in dependency_tracker.resources.get('floating_ip', []):
            fip = FloatingIpSM.get(fip_id)
            if fip:
                for vmi_id in fip.virtual_machine_interfaces:
                    vmi = VirtualMachineInterfaceSM.get(vmi_id)
                    if vmi and vmi.virtual_ip:
                        self.netns_manager.add_fip_to_vip_vmi(vmi, fip)

        for lr_id in dependency_tracker.resources.get('logical_router', []):
            lr = LogicalRouterSM.get(lr_id)
            if lr:
                self.snat_agent.update_snat_instance(lr)

    def post_init(self, vnc_lib, args=None):
        # api server
        self._vnc_lib = vnc_lib

        self._nova_client = importutils.import_object(
            'svc_monitor.nova_client.ServiceMonitorNovaClient',
            self._args, self.logger)

        # load vrouter scheduler
        self.vrouter_scheduler = importutils.import_object(
            self._args.si_netns_scheduler_driver,
            self._vnc_lib, self._nova_client,
            self._args)

        # load virtual machine instance manager
        self.vm_manager = importutils.import_object(
            'svc_monitor.virtual_machine_manager.VirtualMachineManager',
            self._vnc_lib, self.si_db, self.logger,
            self.vrouter_scheduler, self._nova_client, self._args)

        # load network namespace instance manager
        self.netns_manager = importutils.import_object(
            'svc_monitor.instance_manager.NetworkNamespaceManager',
            self._vnc_lib, self.si_db, self.logger,
            self.vrouter_scheduler, self._nova_client, self._args)

        # load a vrouter instance manager
        self.vrouter_manager = importutils.import_object(
            'svc_monitor.vrouter_instance_manager.VRouterInstanceManager',
            self._vnc_lib, self.si_db, self.logger,
            self.vrouter_scheduler, self._nova_client, self._args)

        # load a loadbalancer agent
        self.loadbalancer_agent = LoadbalancerAgent(self, self._vnc_lib, self._args)
        self.snat_agent = SNATAgent(self, self._vnc_lib)

        # Read the cassandra and populate the entry in ServiceMonitor DB
        self.sync_sm()

        # create default analyzer template
        self._create_default_template('analyzer-template', 'analyzer',
                                      flavor='m1.medium',
                                      image_name='analyzer')
        # create default NAT template
        self._create_default_template('nat-template', 'firewall',
                                      svc_mode='in-network-nat',
                                      image_name='analyzer',
                                      flavor='m1.medium')
        # create default netns SNAT template
        self._create_default_template('netns-snat-template', 'source-nat',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        # create default loadbalancer template
        self._create_default_template('haproxy-loadbalancer-template', 'loadbalancer',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        self._create_default_template('docker-template', 'firewall',
                                      svc_mode='transparent',
                                      image_name="ubuntu",
                                      hypervisor_type='vrouter-instance',
                                      vrouter_instance_type='docker',
                                      instance_data={
                                          "command": "/bin/bash"
                                      })

        # upgrade handling
        self.upgrade()

        # check services
        self.launch_services()

        self._db_resync_done.set()

    def upgrade(self):
        for si in ServiceInstanceSM.values():
            st = ServiceTemplateSM.get(si.service_template)
            if not st:
                continue

            for vm_id in si.virtual_machines:
                vm = VirtualMachineSM.get(vm_id)
                if vm.virtualization_type:
                    continue
                nova_vm = self._nova_client.oper('servers', 'get',
                    si.proj_name, id=vm_id)
                if not nova_vm:
                    continue

                si_obj = ServiceInstance()
                si_obj.name = si.name
                si_obj.fq_name = si.fq_name
                instance_name = self.vm_manager._get_instance_name(
                    si_obj, vm.index)
                if vm.name == instance_name:
                    continue
                nova_vm.update(name=instance_name)
                vm_obj = VirtualMachine()
                vm_obj.uuid = vm_id
                vm_obj.fq_name = [vm_id]
                vm_obj.set_display_name(instance_name + '__' +
                    st.virtualization_type)
                try:
                    self._vnc_lib.virtual_machine_update(vm_obj)
                except Exception:
                    pass

    def launch_services(self):
        for si in ServiceInstanceSM.values():
            self._create_service_instance(si)

    def sync_sm(self):
        vn_set = set()
        vmi_set = set()
        iip_set = set()
        ok, lb_pool_list = self._cassandra._cassandra_loadbalancer_pool_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in lb_pool_list:
                lb_pool = LoadbalancerPoolSM.locate(uuid)
                if lb_pool.virtual_machine_interface:
                    vmi_set.add(lb_pool.virtual_machine_interface)

        ok, lb_pool_member_list = self._cassandra._cassandra_loadbalancer_member_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in lb_pool_member_list:
                lb_pool_member = LoadbalancerMemberSM.locate(uuid)

        ok, lb_vip_list = self._cassandra._cassandra_virtual_ip_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in lb_vip_list:
                virtual_ip = VirtualIpSM.locate(uuid)
                if virtual_ip.virtual_machine_interface:
                    vmi_set.add(virtual_ip.virtual_machine_interface)

        ok, lb_hm_list = self._cassandra._cassandra_loadbalancer_healthmonitor_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in lb_hm_list:
                lb_hm = HealthMonitorSM.locate(uuid)

        ok, si_list = self._cassandra._cassandra_service_instance_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in si_list:
                si = ServiceInstanceSM.locate(uuid)

        ok, st_list = self._cassandra._cassandra_service_template_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in st_list:
                st = ServiceTemplateSM.locate(uuid)

        ok, vn_list = self._cassandra._cassandra_virtual_network_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in vn_list:
                vn = VirtualNetworkSM.locate(uuid)
                vmi_set |= vn.virtual_machine_interfaces

        ok, ifd_list = self._cassandra._cassandra_physical_interface_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in ifd_list:
                ifd = PhysicalInterfaceSM.locate(uuid)


        ok, ifl_list = self._cassandra._cassandra_logical_interface_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in ifl_list:
                ifl = LogicalInterfaceSM.locate(uuid)
                if ifl.virtual_machine_interface:
                    vmi_set.add(ifl.virtual_machine_interface)

        ok, pr_list = self._cassandra._cassandra_physical_router_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in pr_list:
                pr = PhysicalRouterSM.locate(uuid)

        ok, vr_list = self._cassandra._cassandra_virtual_router_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in vr_list:
                vr = VirtualRouterSM.locate(uuid)

        ok, vmi_list = self._cassandra._cassandra_virtual_machine_interface_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in vmi_list:
                vmi = VirtualMachineInterfaceSM.locate(uuid)
                if vmi.instance_ip:
                    iip_set.add(vmi.instance_ip)

        ok, irt_list = self._cassandra._cassandra_interface_route_table_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in irt_list:
                irt = InterfaceRouteTableSM.locate(uuid)

        ok, project_list = self._cassandra._cassandra_project_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in project_list:
                prj = ProjectSM.locate(uuid)

        ok, sas_list = self._cassandra._cassandra_service_appliance_set_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in sas_list:
                sas = ServiceApplianceSetSM.locate(uuid)

        ok, sa_list = self._cassandra._cassandra_service_appliance_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in sa_list:
                sa = ServiceApplianceSM.locate(uuid)

        ok, domain_list = self._cassandra._cassandra_domain_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in domain_list:
                DomainSM.locate(uuid)

        ok, iip_list = self._cassandra._cassandra_instance_ip_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in iip_list:
                InstanceIpSM.locate(uuid)

        ok, fip_list = self._cassandra._cassandra_floating_ip_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in fip_list:
                FloatingIpSM.locate(uuid)

        ok, sg_list = self._cassandra._cassandra_security_group_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in sg_list:
                SecurityGroupSM.locate(uuid)

        ok, vm_list = self._cassandra._cassandra_virtual_machine_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in vm_list:
                vm = VirtualMachineSM.locate(uuid)
                if vm.service_instance:
                    continue
                for vmi_id in vm.virtual_machine_interfaces:
                    vmi = VirtualMachineInterfaceSM.get(vmi_id)
                    if not vmi:
                        continue
                    self.check_link_si_to_vm(vm, vmi)


        ok, lr_list = self._cassandra._cassandra_logical_router_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in lr_list:
                lr = LogicalRouterSM.locate(uuid)

        # Load the loadbalancer driver
        self.loadbalancer_agent.load_drivers()

        for lb_pool in LoadbalancerPoolSM.values():
            lb_pool.add()

        # Audit the lb pools
        self.loadbalancer_agent.audit_lb_pools()

        # Audit the SNAT instances
        self.snat_agent.audit_snat_instances()

    # end sync_sm

    # create service template
    def _create_default_template(self, st_name, svc_type, svc_mode=None,
                                 hypervisor_type='virtual-machine',
                                 image_name=None, flavor=None, scaling=False,
                                 vrouter_instance_type=None,
                                 instance_data=None):
        domain_name = 'default-domain'
        domain_fq_name = [domain_name]
        st_fq_name = [domain_name, st_name]
        self.logger.log_info("Creating %s %s hypervisor %s" %
            (domain_name, st_name, hypervisor_type))

        domain_obj = None
        for domain in DomainSM.values():
            if domain.fq_name == domain_fq_name:
                domain_obj = Domain()
                domain_obj.uuid = domain.uuid
                domain_obj.fq_name = domain_fq_name
                break
        if not domain_obj:
            self.logger.log_error("%s domain not found" % (domain_name))
            return

        for st in ServiceTemplateSM.values():
            if st.fq_name == st_fq_name:
                self.logger.log_info("%s exists uuid %s" %
                    (st.name, str(st.uuid)))
                return

        svc_properties = ServiceTemplateType()
        svc_properties.set_service_type(svc_type)
        svc_properties.set_service_mode(svc_mode)
        svc_properties.set_service_virtualization_type(hypervisor_type)
        svc_properties.set_image_name(image_name)
        svc_properties.set_flavor(flavor)
        svc_properties.set_ordered_interfaces(True)
        svc_properties.set_service_scaling(scaling)

        # set interface list
        if svc_type == 'analyzer':
            if_list = [['left', False]]
        elif hypervisor_type == 'network-namespace':
            if_list = [['right', True], ['left', True]]
        else:
            if_list = [
                ['management', False], ['left', False], ['right', False]]

        for itf in if_list:
            if_type = ServiceTemplateInterfaceType(shared_ip=itf[1])
            if_type.set_service_interface_type(itf[0])
            svc_properties.add_interface_type(if_type)

        if vrouter_instance_type is not None:
            svc_properties.set_vrouter_instance_type(vrouter_instance_type)

        if instance_data is not None:
            svc_properties.set_instance_data(
                json.dumps(instance_data, separators=(',', ':')))

        st_obj = ServiceTemplate(name=st_name, domain_obj=domain)
        st_obj.set_service_template_properties(svc_properties)
        try:
            st_uuid = self._vnc_lib.service_template_create(st_obj)
        except Exception as e:
            self.logger.log_error("%s create failed with error %s" %
                (st_name, str(e)))
            return

        self.logger.log_info("%s created with uuid %s" %
            (st_name, str(st_uuid)))
    #_create_default_analyzer_template

    def check_link_si_to_vm(self, vm, vmi):
        if vm.service_instance:
            return
        if not vmi.if_type:
            return

        si_fq_name = vmi.name.split('__')[0:3]
        index = int(vmi.name.split('__')[3]) - 1
        for si in ServiceInstanceSM.values():
            if si.fq_name != si_fq_name:
                continue
            st = ServiceTemplateSM.get(si.service_template)
            self.vm_manager.link_si_to_vm(si, st, index, vm.uuid)
            return

    def _create_service_instance(self, si):
        if si.state == 'active':
            return
        st = ServiceTemplateSM.get(si.service_template)
        if not st:
            self.logger.log_error("template not found for %s" %
                ((':').join(si.fq_name)))
            return

        try:
            if st.virtualization_type == 'virtual-machine':
                self.vm_manager.create_service(st, si)
            elif st.virtualization_type == 'network-namespace':
                self.netns_manager.create_service(st, si)
            elif st.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.create_service(st, si)
            else:
                self.logger.log_error("Unknown virt type: %s" %
                    st.virtualization_type)
        except Exception:
            cgitb_error_log(self)

    def _delete_service_instance(self, vm):
        self.logger.log_info("Deleting VM %s %s" %
            ((':').join(vm.proj_fq_name), vm.uuid))

        try:
            if vm.virtualization_type == svc_info.get_vm_instance_type():
                self.vm_manager.delete_service(vm)
            elif vm.virtualization_type == svc_info.get_netns_instance_type():
                self.netns_manager.delete_service(vm)
            elif vm.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.delete_service(vm)
        except Exception:
            cgitb_error_log(self)

        # generate UVE
        si_fq_name = vm.display_name.split('__')[:-2]
        si_fq_str = (':').join(si_fq_name)
        self.logger.uve_svc_instance(si_fq_str, status='DELETE',
                                     vms=[{'uuid': vm.uuid}])
        return True

    def _relaunch_service_instance(self, si):
        if si.state == 'active':
            si.state = 'relaunch'
            self._create_service_instance(si)

    def _check_service_running(self, si):
        if si.state != 'active':
            return
        st = ServiceTemplateSM.get(si.service_template)
        if st.virtualization_type == 'virtual-machine':
            status = self.vm_manager.check_service(si)
        elif st.virtualization_type == 'network-namespace':
            status = self.netns_manager.check_service(si)
        elif st.virtualization_type == 'vrouter-instance':
            status = self.vrouter_manager.check_service(si)

        return status

    def _delete_interface_route_table(self, irt_uuid):
        try:
            self._vnc_lib.interface_route_table_delete(id=irt_uuid)
        except (NoIdError, RefsExistError):
            return

    def _delete_shared_vn(self, vn_uuid):
        try:
            self.logger.log_info("Deleting vn %s" % (vn_uuid))
            self._vnc_lib.virtual_network_delete(id=vn_uuid)
        except (NoIdError, RefsExistError):
            pass

    @staticmethod
    def reset():
        for cls in DBBase._OBJ_TYPE_MAP.values():
            cls.reset()
Example #5
0
class SvcMonitor(object):
    """
    data + methods used/referred to by ssrc and arc greenlets
    """
    _REACTION_MAP = {
        "service_appliance_set": {
            'self': [],
            'service_appliance': []
        },
        "service_appliance": {
            'self': ['service_appliance_set'],
            'service_appliance_set': []
        },
        "loadbalancer_pool": {
            'self': [],
            'virtual_ip': [],
            'loadbalancer_member': [],
            'loadbalancer_healthmonitor': [],
        },
        "loadbalancer_member": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "virtual_ip": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "loadbalancer_healthmonitor": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "service_instance": {
            'self': ['virtual_machine'],
            'virtual_machine': []
        },
        "instance_ip": {
            'self': [],
        },
        "floating_ip": {
            'self': [],
        },
        "service_template": {
            'self': [],
        },
        "physical_router": {
            'self': [],
        },
        "physical_interface": {
            'self': [],
        },
        "logical_interface": {
            'self': [],
        },
        "virtual_network": {
            'self': [],
        },
        "virtual_machine": {
            'self': ['virtual_machine_interface'],
            'service_instance': [],
            'virtual_machine_interface': [],
        },
        "virtual_machine_interface": {
            'self': ['interface_route_table', 'virtual_machine'],
            'interface_route_table': [],
            'virtual_machine': [],
        },
        "interface_route_table": {
            'self': [],
            'virtual_machine_interface': [],
        },
        "project": {
            'self': [],
        },
    }

    def __init__(self, args=None):
        self._args = args

        # create database and logger
        self.si_db = ServiceInstanceDB(args)

        # initialize discovery client
        self._disc = None
        if self._args.disc_server_ip and self._args.disc_server_port:
            self._disc = client.DiscoveryClient(
                self._args.disc_server_ip, self._args.disc_server_port,
                ModuleNames[Module.SVC_MONITOR])

        # initialize logger
        self.logger = ServiceMonitorLogger(self.si_db, self._disc, args)
        self.si_db.add_logger(self.logger)
        self.si_db.init_database()

        # rotating log file for catchall errors
        self._err_file = self._args.trace_file
        self._svc_err_logger = logging.getLogger('SvcErrLogger')
        self._svc_err_logger.setLevel(logging.ERROR)
        try:
            with open(self._err_file, 'a'):
                handler = logging.handlers.RotatingFileHandler(self._err_file,
                                                               maxBytes=64 *
                                                               1024,
                                                               backupCount=2)
                self._svc_err_logger.addHandler(handler)
        except IOError:
            self.logger.log_warning("Failed to open trace file %s" %
                                    self._err_file)

        # Connect to Rabbit and Initialize cassandra connection
        self._connect_rabbit()

    def _connect_rabbit(self):
        rabbit_server = self._args.rabbit_server
        rabbit_port = self._args.rabbit_port
        rabbit_user = self._args.rabbit_user
        rabbit_password = self._args.rabbit_password
        rabbit_vhost = self._args.rabbit_vhost
        rabbit_ha_mode = self._args.rabbit_ha_mode

        self._db_resync_done = gevent.event.Event()

        q_name = 'svc_mon.%s' % (socket.gethostname())
        self._vnc_kombu = VncKombuClient(rabbit_server, rabbit_port,
                                         rabbit_user, rabbit_password,
                                         rabbit_vhost, rabbit_ha_mode, q_name,
                                         self._vnc_subscribe_callback,
                                         self.config_log)

        cass_server_list = self._args.cassandra_server_list
        reset_config = self._args.reset_config
        self._cassandra = VncCassandraClient(cass_server_list, reset_config,
                                             self._args.cluster_id, None,
                                             self.config_log)
        DBBase.init(self, self.logger, self._cassandra)

    # end _connect_rabbit

    def config_log(self, msg, level):
        self.logger.log(msg)

    def _vnc_subscribe_callback(self, oper_info):
        self._db_resync_done.wait()
        try:
            msg = "Notification Message: %s" % (pformat(oper_info))
            self.config_log(msg, level=SandeshLevel.SYS_DEBUG)
            obj_type = oper_info['type'].replace('-', '_')
            obj_class = DBBase._OBJ_TYPE_MAP.get(obj_type)
            if obj_class is None:
                return

            if oper_info['oper'] == 'CREATE' or oper_info['oper'] == 'UPDATE':
                dependency_tracker = DependencyTracker(DBBase._OBJ_TYPE_MAP,
                                                       self._REACTION_MAP)
                obj_id = oper_info['uuid']
                obj = obj_class.get(obj_id)
                if obj is not None:
                    dependency_tracker.evaluate(obj_type, obj)
                else:
                    obj = obj_class.locate(obj_id)
                obj.update()
                dependency_tracker.evaluate(obj_type, obj)
            elif oper_info['oper'] == 'DELETE':
                obj_id = oper_info['uuid']
                obj = obj_class.get(obj_id)
                if obj is None:
                    return
                dependency_tracker = DependencyTracker(DBBase._OBJ_TYPE_MAP,
                                                       self._REACTION_MAP)
                dependency_tracker.evaluate(obj_type, obj)
                obj_class.delete(obj_id)
            else:
                # unknown operation
                self.config_log('Unknown operation %s' % oper_info['oper'],
                                level=SandeshLevel.SYS_ERR)
                return

            if obj is None:
                self.config_log('Error while accessing %s uuid %s' %
                                (obj_type, obj_id))
                return

        except Exception:
            string_buf = cStringIO.StringIO()
            cgitb.Hook(file=string_buf, format="text").handle(sys.exc_info())
            self.config_log(string_buf.getvalue(), level=SandeshLevel.SYS_ERR)

        for sas_id in dependency_tracker.resources.get('service_appliance_set',
                                                       []):
            sas_obj = ServiceApplianceSetSM.get(sas_id)
            if sas_obj is not None:
                sas_obj.add()

        for lb_pool_id in dependency_tracker.resources.get(
                'loadbalancer_pool', []):
            lb_pool = LoadbalancerPoolSM.get(lb_pool_id)
            if lb_pool is not None:
                lb_pool.add()

        for si_id in dependency_tracker.resources.get('service_instance', []):
            si = ServiceInstanceSM.get(si_id)
            if si:
                self._create_service_instance(si)
            else:
                for vm_id in dependency_tracker.resources.get(
                        'virtual_machine', []):
                    vm = VirtualMachineSM.get(vm_id)
                    self._delete_service_instance(vm)

        for vn_id in dependency_tracker.resources.get('virtual_network', []):
            vn = VirtualNetworkSM.get(vn_id)
            if vn:
                for si_id in ServiceInstanceSM:
                    si = ServiceInstanceSM.get(si_id)
                    if (':').join(vn.fq_name) in si.params.values():
                        self._create_service_instance(si)

        for vmi_id in dependency_tracker.resources.get(
                'virtual_machine_interface', []):
            vmi = VirtualMachineInterfaceSM.get(vmi_id)
            if vmi:
                for vm_id in dependency_tracker.resources.get(
                        'virtual_machine', []):
                    vm = VirtualMachineSM.get(vm_id)
                    if vm:
                        self.check_link_si_to_vm(vm, vmi)
            else:
                for irt_id in dependency_tracker.resources.get(
                        'interface_route_table', []):
                    self._delete_interface_route_table(irt_id)

        for fip_id in dependency_tracker.resources.get('floating_ip', []):
            fip = FloatingIpSM.get(fip_id)
            if fip:
                for vmi_id in fip.virtual_machine_interfaces:
                    vmi = VirtualMachineInterfaceSM.get(vmi_id)
                    if vmi and vmi.virtual_ip:
                        self.netns_manager.add_fip_to_vip_vmi(vmi, fip)

    def post_init(self, vnc_lib, args=None):
        # api server
        self._vnc_lib = vnc_lib

        self._nova_client = importutils.import_object(
            'svc_monitor.nova_client.ServiceMonitorNovaClient', self._args,
            self.logger)

        # load vrouter scheduler
        self.vrouter_scheduler = importutils.import_object(
            self._args.si_netns_scheduler_driver, self._vnc_lib,
            self._nova_client, self._args)

        # load virtual machine instance manager
        self.vm_manager = importutils.import_object(
            'svc_monitor.virtual_machine_manager.VirtualMachineManager',
            self._vnc_lib, self.si_db, self.logger, self.vrouter_scheduler,
            self._nova_client, self._args)

        # load network namespace instance manager
        self.netns_manager = importutils.import_object(
            'svc_monitor.instance_manager.NetworkNamespaceManager',
            self._vnc_lib, self.si_db, self.logger, self.vrouter_scheduler,
            self._nova_client, self._args)

        # load a vrouter instance manager
        self.vrouter_manager = importutils.import_object(
            'svc_monitor.vrouter_instance_manager.VRouterInstanceManager',
            self._vnc_lib, self.si_db, self.logger, self.vrouter_scheduler,
            self._nova_client, self._args)

        # load a loadbalancer agent
        self.loadbalancer_agent = LoadbalancerAgent(self, self._vnc_lib,
                                                    self._args)

        # Read the cassandra and populate the entry in ServiceMonitor DB
        self.sync_sm()

        # create default analyzer template
        self._create_default_template('analyzer-template',
                                      'analyzer',
                                      flavor='m1.medium',
                                      image_name='analyzer')
        # create default NAT template
        self._create_default_template('nat-template',
                                      'firewall',
                                      svc_mode='in-network-nat',
                                      image_name='analyzer',
                                      flavor='m1.medium')
        # create default netns SNAT template
        self._create_default_template('netns-snat-template',
                                      'source-nat',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        # create default loadbalancer template
        self._create_default_template('haproxy-loadbalancer-template',
                                      'loadbalancer',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        self._create_default_template('docker-template',
                                      'firewall',
                                      svc_mode='transparent',
                                      image_name="ubuntu",
                                      hypervisor_type='vrouter-instance',
                                      vrouter_instance_type='docker',
                                      instance_data={"command": "/bin/bash"})

        # upgrade handling
        self.upgrade()

        # check services
        self.launch_services()

        self._db_resync_done.set()

    def upgrade(self):
        for si in ServiceInstanceSM.values():
            st = ServiceTemplateSM.get(si.service_template)
            if not st:
                continue
            for vm_id in si.virtual_machines:
                vm = VirtualMachineSM.get(vm_id)
                if vm.virtualization_type:
                    continue
                nova_vm = self._nova_client.oper('servers',
                                                 'get',
                                                 si.proj_name,
                                                 id=vm_id)
                if not nova_vm:
                    continue

                si_obj = ServiceInstance()
                si_obj.name = si.name
                si_obj.fq_name = si.fq_name
                instance_name = self.vm_manager._get_instance_name(
                    si_obj, vm.index)
                if vm.name == instance_name:
                    continue
                nova_vm.update(name=instance_name)
                vm_obj = VirtualMachine()
                vm_obj.uuid = vm_id
                vm_obj.fq_name = [vm_id]
                vm_obj.set_display_name(instance_name + '__' +
                                        st.virtualization_type)
                try:
                    self._vnc_lib.virtual_machine_update(vm_obj)
                except Exception:
                    pass

    def launch_services(self):
        for si in ServiceInstanceSM.values():
            self._create_service_instance(si)

    def sync_sm(self):
        vn_set = set()
        vmi_set = set()
        iip_set = set()
        ok, lb_pool_list = self._cassandra._cassandra_loadbalancer_pool_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in lb_pool_list:
                lb_pool = LoadbalancerPoolSM.locate(uuid)
                if lb_pool.virtual_machine_interface:
                    vmi_set.add(lb_pool.virtual_machine_interface)

        ok, lb_pool_member_list = self._cassandra._cassandra_loadbalancer_member_list(
        )
        if not ok:
            pass
        else:
            for fq_name, uuid in lb_pool_member_list:
                lb_pool_member = LoadbalancerMemberSM.locate(uuid)

        ok, lb_vip_list = self._cassandra._cassandra_virtual_ip_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in lb_vip_list:
                virtual_ip = VirtualIpSM.locate(uuid)
                if virtual_ip.virtual_machine_interface:
                    vmi_set.add(virtual_ip.virtual_machine_interface)

        ok, lb_hm_list = self._cassandra._cassandra_loadbalancer_healthmonitor_list(
        )
        if not ok:
            pass
        else:
            for fq_name, uuid in lb_hm_list:
                lb_hm = HealthMonitorSM.locate(uuid)

        ok, si_list = self._cassandra._cassandra_service_instance_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in si_list:
                si = ServiceInstanceSM.locate(uuid)

        ok, st_list = self._cassandra._cassandra_service_template_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in st_list:
                st = ServiceTemplateSM.locate(uuid)

        ok, vn_list = self._cassandra._cassandra_virtual_network_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in vn_list:
                vn = VirtualNetworkSM.locate(uuid)
                vmi_set |= vn.virtual_machine_interfaces

        ok, ifd_list = self._cassandra._cassandra_physical_interface_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in ifd_list:
                ifd = PhysicalInterfaceSM.locate(uuid)

        ok, ifl_list = self._cassandra._cassandra_logical_interface_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in ifl_list:
                ifl = LogicalInterfaceSM.locate(uuid)
                if ifl.virtual_machine_interface:
                    vmi_set.add(ifl.virtual_machine_interface)

        ok, pr_list = self._cassandra._cassandra_physical_router_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in pr_list:
                pr = PhysicalRouterSM.locate(uuid)

        ok, vr_list = self._cassandra._cassandra_virtual_router_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in vr_list:
                vr = VirtualRouterSM.locate(uuid)

        ok, vmi_list = self._cassandra._cassandra_virtual_machine_interface_list(
        )
        if not ok:
            pass
        else:
            for fq_name, uuid in vmi_list:
                vmi = VirtualMachineInterfaceSM.locate(uuid)
                if vmi.instance_ip:
                    iip_set.add(vmi.instance_ip)

        ok, irt_list = self._cassandra._cassandra_interface_route_table_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in irt_list:
                irt = InterfaceRouteTableSM.locate(uuid)

        ok, project_list = self._cassandra._cassandra_project_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in project_list:
                prj = ProjectSM.locate(uuid)

        ok, sas_list = self._cassandra._cassandra_service_appliance_set_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in sas_list:
                sas = ServiceApplianceSetSM.locate(uuid)

        ok, sa_list = self._cassandra._cassandra_service_appliance_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in sa_list:
                sa = ServiceApplianceSM.locate(uuid)

        ok, domain_list = self._cassandra._cassandra_domain_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in domain_list:
                DomainSM.locate(uuid)

        ok, iip_list = self._cassandra._cassandra_instance_ip_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in iip_list:
                InstanceIpSM.locate(uuid)

        ok, fip_list = self._cassandra._cassandra_floating_ip_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in fip_list:
                FloatingIpSM.locate(uuid)

        ok, sg_list = self._cassandra._cassandra_security_group_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in sg_list:
                SecurityGroupSM.locate(uuid)

        ok, vm_list = self._cassandra._cassandra_virtual_machine_list()
        if not ok:
            pass
        else:
            for fq_name, uuid in vm_list:
                vm = VirtualMachineSM.locate(uuid)
                if vm.service_instance:
                    continue
                for vmi_id in vm.virtual_machine_interfaces:
                    vmi = VirtualMachineInterfaceSM.get(vmi_id)
                    if not vmi:
                        continue
                    self.check_link_si_to_vm(vm, vmi)

        # Load the loadbalancer driver
        self.loadbalancer_agent.load_drivers()

        for lb_pool in LoadbalancerPoolSM.values():
            lb_pool.add()

        # Audit the lb pools
        self.loadbalancer_agent.audit_lb_pools()

    # end sync_sm

    # create service template
    def _create_default_template(self,
                                 st_name,
                                 svc_type,
                                 svc_mode=None,
                                 hypervisor_type='virtual-machine',
                                 image_name=None,
                                 flavor=None,
                                 scaling=False,
                                 vrouter_instance_type=None,
                                 instance_data=None):
        domain_name = 'default-domain'
        domain_fq_name = [domain_name]
        st_fq_name = [domain_name, st_name]
        self.logger.log_info("Creating %s %s hypervisor %s" %
                             (domain_name, st_name, hypervisor_type))

        domain_obj = None
        for domain in DomainSM.values():
            if domain.fq_name == domain_fq_name:
                domain_obj = Domain()
                domain_obj.uuid = domain.uuid
                domain_obj.fq_name = domain_fq_name
                break
        if not domain_obj:
            self.logger.log_error("%s domain not found" % (domain_name))
            return

        for st in ServiceTemplateSM.values():
            if st.fq_name == st_fq_name:
                self.logger.log_info("%s exists uuid %s" %
                                     (st.name, str(st.uuid)))
                return

        svc_properties = ServiceTemplateType()
        svc_properties.set_service_type(svc_type)
        svc_properties.set_service_mode(svc_mode)
        svc_properties.set_service_virtualization_type(hypervisor_type)
        svc_properties.set_image_name(image_name)
        svc_properties.set_flavor(flavor)
        svc_properties.set_ordered_interfaces(True)
        svc_properties.set_service_scaling(scaling)

        # set interface list
        if svc_type == 'analyzer':
            if_list = [['left', False]]
        elif hypervisor_type == 'network-namespace':
            if_list = [['right', True], ['left', True]]
        else:
            if_list = [['management', False], ['left', False],
                       ['right', False]]

        for itf in if_list:
            if_type = ServiceTemplateInterfaceType(shared_ip=itf[1])
            if_type.set_service_interface_type(itf[0])
            svc_properties.add_interface_type(if_type)

        if vrouter_instance_type is not None:
            svc_properties.set_vrouter_instance_type(vrouter_instance_type)

        if instance_data is not None:
            svc_properties.set_instance_data(
                json.dumps(instance_data, separators=(',', ':')))

        st_obj = ServiceTemplate(name=st_name, domain_obj=domain)
        st_obj.set_service_template_properties(svc_properties)
        try:
            st_uuid = self._vnc_lib.service_template_create(st_obj)
        except Exception as e:
            print e

        self.logger.log_info("%s created with uuid %s" %
                             (st_name, str(st_uuid)))

    #_create_default_analyzer_template

    def check_link_si_to_vm(self, vm, vmi):
        if vm.service_instance:
            return
        if not vmi.if_type:
            return

        si_fq_name = vmi.name.split('__')[0:3]
        index = int(vmi.name.split('__')[3]) - 1
        for si in ServiceInstanceSM.values():
            if si.fq_name != si_fq_name:
                continue
            st = ServiceTemplateSM.get(si.service_template)
            self.vm_manager.link_si_to_vm(si, st, index, vm.uuid)
            return

    def _create_service_instance(self, si):
        if si.state == 'active':
            return
        st = ServiceTemplateSM.get(si.service_template)
        if not st:
            self.logger.log_error("template not found for %s" %
                                  ((':').join(si.fq_name)))
            return

        try:
            if st.virtualization_type == 'virtual-machine':
                self.vm_manager.create_service(st, si)
            elif st.virtualization_type == 'network-namespace':
                self.netns_manager.create_service(st, si)
            elif st.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.create_service(st, si)
            else:
                self.logger.log_error("Unknown virt type: %s" %
                                      st.virtualization_type)
        except Exception:
            cgitb_error_log(self)

    def _delete_service_instance(self, vm):
        self.logger.log_info("Deleting VM %s %s" %
                             ((':').join(vm.proj_fq_name), vm.uuid))

        try:
            if vm.virtualization_type == svc_info.get_vm_instance_type():
                self.vm_manager.delete_service(vm)
            elif vm.virtualization_type == svc_info.get_netns_instance_type():
                self.netns_manager.delete_service(vm)
            elif vm.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.delete_service(vm)
        except Exception:
            cgitb_error_log(self)

        # generate UVE
        si_fq_name = vm.display_name.split('__')[:-2]
        si_fq_str = (':').join(si_fq_name)
        self.logger.uve_svc_instance(si_fq_str,
                                     status='DELETE',
                                     vms=[{
                                         'uuid': vm.uuid
                                     }])
        return True

    def _relaunch_service_instance(self, si):
        if si.state == 'active':
            si.state = 'relaunch'
            self._create_service_instance(si)

    def _check_service_running(self, si):
        if si.state != 'active':
            return
        st = ServiceTemplateSM.get(si.service_template)
        if st.virtualization_type == 'virtual-machine':
            status = self.vm_manager.check_service(si)
        elif st.virtualization_type == 'network-namespace':
            status = self.netns_manager.check_service(si)
        elif st.virtualization_type == 'vrouter-instance':
            status = self.vrouter_manager.check_service(si)

        return status

    def _delete_interface_route_table(self, irt_uuid):
        try:
            self._vnc_lib.interface_route_table_delete(id=irt_uuid)
        except (NoIdError, RefsExistError):
            return

    def _delete_shared_vn(self, vn_uuid):
        try:
            self.logger.log_info("Deleting vn %s" % (vn_uuid))
            self._vnc_lib.virtual_network_delete(id=vn_uuid)
        except (NoIdError, RefsExistError):
            pass

    @staticmethod
    def reset():
        for cls in DBBase._OBJ_TYPE_MAP.values():
            cls.reset()
Example #6
0
class SvcMonitor(object):
    def __init__(self, sm_logger=None, args=None):
        self._args = args
        self._args.analytics_api_ssl_params = analytics_api_ssl_params(args)
        # initialize logger
        if sm_logger is not None:
            self.logger = sm_logger
        else:
            # Initialize logger
            self.logger = ServiceMonitorLogger(args)

        # init object_db
        self._object_db = ServiceMonitorDB(self._args, self.logger)
        DBBaseSM.init(self, self.logger, self._object_db)

        # init rabbit connection
        rabbitmq_cfg = get_rabbitmq_cfg(args)
        if 'host_ip' in self._args:
            host_ip = self._args.host_ip
        else:
            host_ip = socket.gethostbyname(socket.getfqdn())
        self.rabbit = VncAmqpHandle(self.logger._sandesh, self.logger,
                                    DBBaseSM, REACTION_MAP, 'svc_monitor',
                                    rabbitmq_cfg, host_ip,
                                    self._args.trace_file)
        self.rabbit.establish()

    def post_init(self, vnc_lib, args=None):
        # api server
        self._vnc_lib = vnc_lib

        try:
            self._nova_client = importutils.import_object(
                'svc_monitor.nova_client.ServiceMonitorNovaClient', self._args,
                self.logger)
        except Exception as e:
            self._nova_client = None

        # agent manager
        self._agent_manager = AgentManager()

        # load vrouter scheduler
        self.vrouter_scheduler = importutils.import_object(
            self._args.si_netns_scheduler_driver, self._vnc_lib,
            self._nova_client, None, self.logger, self._args)

        # load virtual machine instance manager
        self.vm_manager = importutils.import_object(
            'svc_monitor.virtual_machine_manager.VirtualMachineManager',
            self._vnc_lib, self._object_db, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load network namespace instance manager
        self.netns_manager = importutils.import_object(
            'svc_monitor.instance_manager.NetworkNamespaceManager',
            self._vnc_lib, self._object_db, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load a vrouter instance manager
        self.vrouter_manager = importutils.import_object(
            'svc_monitor.vrouter_instance_manager.VRouterInstanceManager',
            self._vnc_lib, self._object_db, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load PNF instance manager
        self.ps_manager = importutils.import_object(
            'svc_monitor.physical_service_manager.PhysicalServiceManager',
            self._vnc_lib, self._object_db, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load a loadbalancer agent
        self.loadbalancer_agent = LoadbalancerAgent(self, self._vnc_lib,
                                                    self._object_db,
                                                    self._args)
        self._agent_manager.register_agent(self.loadbalancer_agent)

        # load a snat agent
        self.snat_agent = SNATAgent(self, self._vnc_lib, self._object_db,
                                    self._args,
                                    ServiceMonitorModuleLogger(self.logger))
        self._agent_manager.register_agent(self.snat_agent)

        # load port tuple agent
        self.port_tuple_agent = PortTupleAgent(
            self, self._vnc_lib, self._object_db, self._args,
            ServiceMonitorModuleLogger(self.logger))
        self._agent_manager.register_agent(self.port_tuple_agent)

        # Read the object_db and populate the entry in ServiceMonitor DB
        self.sync_sm()

        # create default analyzer template
        self._create_default_template('analyzer-template',
                                      'analyzer',
                                      flavor='m1.medium',
                                      image_name='analyzer')
        # create default NAT template
        self._create_default_template('nat-template',
                                      'firewall',
                                      svc_mode='in-network-nat',
                                      image_name='analyzer',
                                      flavor='m1.medium')
        # create default netns SNAT template
        self._create_default_template('netns-snat-template',
                                      'source-nat',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        # create default loadbalancer template
        self._create_default_template('haproxy-loadbalancer-template',
                                      'loadbalancer',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        self._create_default_template('docker-template',
                                      'firewall',
                                      svc_mode='transparent',
                                      image_name="ubuntu",
                                      hypervisor_type='vrouter-instance',
                                      vrouter_instance_type='docker',
                                      instance_data={"command": "/bin/bash"})

        # upgrade handling
        self.upgrade()

        # check services
        self.vrouter_scheduler.vrouters_running()
        self.launch_services()

        self.rabbit._db_resync_done.set()

    def _upgrade_instance_ip(self, vm):
        for vmi_id in vm.virtual_machine_interfaces:
            vmi = VirtualMachineInterfaceSM.get(vmi_id)
            if not vmi:
                continue

            for iip_id in vmi.instance_ips:
                iip = InstanceIpSM.get(iip_id)
                if not iip or iip.service_instance_ip:
                    continue
                iip_obj = InstanceIp()
                iip_obj.name = iip.name
                iip_obj.uuid = iip.uuid
                iip_obj.set_service_instance_ip(True)
                try:
                    self._vnc_lib.instance_ip_update(iip_obj)
                except NoIdError:
                    self.logger.error(
                        "upgrade instance ip to service ip failed %s" %
                        (iip.name))
                    continue

    def _upgrade_auto_policy(self, si, st):
        if st.name != 'netns-snat-template':
            return
        if not si.params['auto_policy']:
            return

        si_obj = ServiceInstance()
        si_obj.uuid = si.uuid
        si_obj.fq_name = si.fq_name
        si_props = ServiceInstanceType(**si.params)
        si_props.set_auto_policy(False)
        si_obj.set_service_instance_properties(si_props)
        try:
            self._vnc_lib.service_instance_update(si_obj)
            self.logger.notice("snat policy upgraded for %s" % (si.name))
        except NoIdError:
            self.logger.error("snat policy upgrade failed for %s" % (si.name))
            return

    def upgrade(self):
        for lr in LogicalRouterSM.values():
            self.snat_agent.upgrade(lr)

        for si in ServiceInstanceSM.values():
            st = ServiceTemplateSM.get(si.service_template)
            if not st:
                continue

            self._upgrade_auto_policy(si, st)

            vm_id_list = list(si.virtual_machines)
            for vm_id in vm_id_list:
                vm = VirtualMachineSM.get(vm_id)
                self._upgrade_instance_ip(vm)
                if vm.virtualization_type:
                    continue

                try:
                    nova_vm = self._nova_client.oper('servers',
                                                     'get',
                                                     si.proj_name,
                                                     id=vm_id)
                except nc_exc.NotFound:
                    nova_vm = None

                if nova_vm:
                    vm_name = nova_vm.name
                    vm.proj_fq_name = nova_vm.name.split('__')[0:2]
                else:
                    vm_name = vm.name

                if not vm_name.split('__')[-1].isdigit():
                    continue

                vm.virtualization_type = st.virtualization_type
                self.delete_service_instance(vm)

    def launch_services(self):
        for si in ServiceInstanceSM.values():
            self.create_service_instance(si)

    def sync_sm(self):
        # Read and Sync all DBase
        for cls in DBBaseSM.get_obj_type_map().values():
            for obj in cls.list_obj():
                cls.locate(obj['uuid'], obj)

        # Link SI and VM
        for vm in VirtualMachineSM.values():
            if vm.service_instance:
                continue
            for vmi_id in vm.virtual_machine_interfaces:
                vmi = VirtualMachineInterfaceSM.get(vmi_id)
                if not vmi:
                    continue
                self.port_delete_or_si_link(vm, vmi)

        # invoke port tuple handling
        try:
            self.port_tuple_agent.update_port_tuples()
        except Exception:
            cgitb_error_log(self)

        # Load the loadbalancer driver
        self.loadbalancer_agent.load_drivers()

        # Invoke the health monitors
        for hm in HealthMonitorSM.values():
            hm.sync()

        # Invoke the loadbalancers
        for lb in LoadbalancerSM.values():
            lb.sync()

        # Invoke the loadbalancer listeners
        for lb_listener in LoadbalancerListenerSM.values():
            lb_listener.sync()

        # Invoke the loadbalancer pools
        for lb_pool in LoadbalancerPoolSM.values():
            lb_pool.sync()

        # Audit the lb pools
        self.loadbalancer_agent.audit_lb_pools()

        # Audit the SNAT instances
        self.snat_agent.audit_snat_instances()

    # end sync_sm

    # create service template
    def _create_default_template(self,
                                 st_name,
                                 svc_type,
                                 svc_mode=None,
                                 hypervisor_type='virtual-machine',
                                 image_name=None,
                                 flavor=None,
                                 scaling=False,
                                 vrouter_instance_type=None,
                                 instance_data=None):
        domain_name = 'default-domain'
        domain_fq_name = [domain_name]
        st_fq_name = [domain_name, st_name]
        self.logger.info("Creating %s %s hypervisor %s" %
                         (domain_name, st_name, hypervisor_type))

        domain_obj = None
        for domain in DomainSM.values():
            if domain.fq_name == domain_fq_name:
                domain_obj = Domain()
                domain_obj.uuid = domain.uuid
                domain_obj.fq_name = domain_fq_name
                break
        if not domain_obj:
            self.logger.error("%s domain not found" % (domain_name))
            return

        for st in ServiceTemplateSM.values():
            if st.fq_name == st_fq_name:
                self.logger.info("%s exists uuid %s" % (st.name, str(st.uuid)))
                return

        svc_properties = ServiceTemplateType()
        svc_properties.set_service_type(svc_type)
        svc_properties.set_service_mode(svc_mode)
        svc_properties.set_service_virtualization_type(hypervisor_type)
        svc_properties.set_image_name(image_name)
        svc_properties.set_flavor(flavor)
        svc_properties.set_ordered_interfaces(True)
        svc_properties.set_service_scaling(scaling)

        # set interface list
        if svc_type == 'analyzer':
            if_list = [['left', False]]
        elif hypervisor_type == 'network-namespace':
            if_list = [['right', True], ['left', True]]
        else:
            if_list = [['management', False], ['left', False],
                       ['right', False]]

        for itf in if_list:
            if_type = ServiceTemplateInterfaceType(shared_ip=itf[1])
            if_type.set_service_interface_type(itf[0])
            svc_properties.add_interface_type(if_type)

        if vrouter_instance_type is not None:
            svc_properties.set_vrouter_instance_type(vrouter_instance_type)

        if instance_data is not None:
            svc_properties.set_instance_data(
                json.dumps(instance_data, separators=(',', ':')))

        st_obj = ServiceTemplate(name=st_name, domain_obj=domain)
        st_obj.set_service_template_properties(svc_properties)
        try:
            st_uuid = self._vnc_lib.service_template_create(st_obj)
        except Exception as e:
            self.logger.error("%s create failed with error %s" %
                              (st_name, str(e)))
            return

        # Create the service template in local db
        ServiceTemplateSM.locate(st_uuid)

        self.logger.info("%s created with uuid %s" % (st_name, str(st_uuid)))

    #_create_default_analyzer_template

    def port_delete_or_si_link(self, vm, vmi):
        if vmi.port_tuples:
            return

        if (vmi.service_instances and vmi.virtual_machine == None):
            self.vm_manager.cleanup_svc_vm_ports([vmi.uuid])
            return

        if not vm or vm.service_instance:
            return

        if not vmi.if_type:
            return

        if len(vmi.name.split('__')) < 4:
            return

        for si in ServiceInstanceSM.values():
            # form the vmi prefix name from si fq_name
            vmi_prefix_name = ('__').join(si.fq_name) + '__'
            if vmi_prefix_name not in vmi.name:
                continue
            index = int(vmi.name.split(vmi_prefix_name)[1].split('__')[0]) - 1
            st = ServiceTemplateSM.get(si.service_template)
            self.vm_manager.link_si_to_vm(si, st, index, vm.uuid)
            return

    def create_service_instance(self, si):
        if si.state == 'active':
            return
        st = ServiceTemplateSM.get(si.service_template)
        if not st:
            self.logger.error("template not found for %s" %
                              ((':').join(si.fq_name)))
            return
        if st.params and st.params.get('version', 1) == 2:
            return

        self.logger.info("Creating SI %s (%s)" %
                         ((':').join(si.fq_name), st.virtualization_type))
        try:
            if st.virtualization_type == 'virtual-machine':
                self.vm_manager.create_service(st, si)
            elif st.virtualization_type == 'network-namespace':
                self.netns_manager.create_service(st, si)
            elif st.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.create_service(st, si)
            elif st.virtualization_type == 'physical-device':
                self.ps_manager.create_service(st, si)
            else:
                self.logger.error("Unknown virt type: %s" %
                                  st.virtualization_type)
        except Exception:
            cgitb_error_log(self)
        si.launch_count += 1
        self.logger.info("SI %s creation success" % (':').join(si.fq_name))

    def delete_service_instance(self, vm):
        self.logger.info("Deleting VM %s %s for SI %s" %
                         ((':').join(vm.fq_name), vm.uuid, vm.service_id))

        try:
            if vm.virtualization_type == svc_info.get_vm_instance_type():
                self.vm_manager.delete_service(vm)
            elif vm.virtualization_type == svc_info.get_netns_instance_type():
                self.netns_manager.delete_service(vm)
            elif vm.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.delete_service(vm)
            elif vm.virtualization_type == 'physical-device':
                self.ps_manager.delete_service(vm)
            self.logger.info("Deleted VM %s %s for SI %s" %
                             ((':').join(vm.fq_name), vm.uuid, vm.service_id))
        except Exception:
            cgitb_error_log(self)

        # generate UVE
        si_fq_name = vm.display_name.split('__')[:-2]
        si_fq_str = (':').join(si_fq_name)
        self.logger.uve_svc_instance(si_fq_str,
                                     status='DELETE',
                                     vms=[{
                                         'uuid': vm.uuid
                                     }])
        return True

    def _relaunch_service_instance(self, si):
        si.state = 'relaunch'
        self.create_service_instance(si)

    def _check_service_running(self, si):
        st = ServiceTemplateSM.get(si.service_template)
        if st.params and st.params.get('version', 1) == 2:
            return
        if st.virtualization_type == 'virtual-machine':
            status = self.vm_manager.check_service(si)
        elif st.virtualization_type == 'network-namespace':
            status = self.netns_manager.check_service(si)
        elif st.virtualization_type == 'vrouter-instance':
            status = self.vrouter_manager.check_service(si)
        elif st.virtualization_type == 'physical-device':
            status = self.ps_manager.check_service(si)
        return status

    def delete_interface_route_table(self, irt_uuid):
        try:
            self._vnc_lib.interface_route_table_delete(id=irt_uuid)
            InterfaceRouteTableSM.delete(irt_uuid)
        except (NoIdError, RefsExistError):
            return

    def _delete_shared_vn(self, vn_uuid):
        try:
            self.logger.info("Deleting vn %s" % (vn_uuid))
            self._vnc_lib.virtual_network_delete(id=vn_uuid)
            VirtualNetworkSM.delete(vn_uuid)
        except (NoIdError, RefsExistError):
            pass

    @staticmethod
    def reset():
        for cls in DBBaseSM.get_obj_type_map().values():
            cls.reset()

    def sighup_handler(self):
        if self._conf_file:
            config = ConfigParser.SafeConfigParser()
            config.read(self._conf_file)
            if 'DEFAULTS' in config.sections():
                try:
                    collectors = config.get('DEFAULTS', 'collectors')
                    if type(collectors) is str:
                        collectors = collectors.split()
                        new_chksum = hashlib.md5(
                            "".join(collectors)).hexdigest()
                        if new_chksum != self._chksum:
                            self._chksum = new_chksum
                            config.random_collectors = random.sample(
                                collectors, len(collectors))
                        # Reconnect to achieve load-balance irrespective of list
                        self.logger.sandesh_reconfig_collectors(config)
                except ConfigParser.NoOptionError as e:
                    pass
class SvcMonitor(object):
    """
    data + methods used/referred to by ssrc and arc greenlets
    """
    _REACTION_MAP = {
        "service_appliance_set": {
            'self': [],
            'service_appliance': []
        },
        "service_appliance": {
            'self': ['service_appliance_set', 'physical_interface'],
            'service_appliance_set': []
        },
        "loadbalancer_pool": {
            'self': [],
            'virtual_ip': [],
            'loadbalancer_listener': [],
            'loadbalancer_member': [],
            'loadbalancer_healthmonitor': [],
        },
        "loadbalancer_member": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "virtual_ip": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "loadbalancer_listener": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': [],
            'loadbalancer': []
        },
        "loadbalancer": {
            'self': ['loadbalancer_listener'],
            'loadbalancer_listener': []
        },
        "loadbalancer_healthmonitor": {
            'self': ['loadbalancer_pool'],
            'loadbalancer_pool': []
        },
        "service_instance": {
            'self':
            ['virtual_machine', 'port_tuple', 'virtual_machine_interface'],
            'virtual_machine': [],
            'port_tuple': [],
            'virtual_machine_interface': []
        },
        "instance_ip": {
            'self': [],
        },
        "floating_ip": {
            'self': [],
        },
        "service_template": {
            'self': [],
        },
        "physical_router": {
            'self': [],
        },
        "physical_interface": {
            'self': [],
            'service_appliance': ['virtual_machine_interface'],
            'virtual_machine_interfaces': ['service_appliance'],
        },
        "logical_interface": {
            'self': [],
        },
        "logical_router": {
            'self': [],
        },
        "virtual_network": {
            'self': [],
        },
        "virtual_machine": {
            'self': ['virtual_machine_interface'],
            'service_instance': [],
            'virtual_machine_interface': [],
        },
        "port_tuple": {
            'self': ['virtual_machine_interface'],
            'service_instance': [],
            'virtual_machine_interface': [],
        },
        "virtual_machine_interface": {
            'self': ['interface_route_table', 'virtual_machine', 'port_tuple'],
            'interface_route_table': [],
            'virtual_machine': [],
            'port_tuple': [],
            'service_instance': ['physical_interface'],
            'physical_interface': ['service_instance']
        },
        "interface_route_table": {
            'self': [],
            'virtual_machine_interface': [],
        },
        "project": {
            'self': [],
        },
        "logical_router": {
            'self': [],
        },
    }

    def __init__(self, args=None):
        self._args = args

        # initialize discovery client
        self._disc = None
        if self._args.disc_server_ip and self._args.disc_server_port:
            self._disc = client.DiscoveryClient(
                self._args.disc_server_ip, self._args.disc_server_port,
                ModuleNames[Module.SVC_MONITOR])
        # initialize logger
        self.logger = ServiceMonitorLogger(self._disc, args)

        # rotating log file for catchall errors
        self._err_file = self._args.trace_file
        self._svc_err_logger = logging.getLogger('SvcErrLogger')
        self._svc_err_logger.setLevel(logging.ERROR)
        try:
            with open(self._err_file, 'a'):
                handler = logging.handlers.RotatingFileHandler(self._err_file,
                                                               maxBytes=64 *
                                                               1024,
                                                               backupCount=2)
                self._svc_err_logger.addHandler(handler)
        except IOError:
            self.logger.log_warning("Failed to open trace file %s" %
                                    self._err_file)

        # Connect to Rabbit and Initialize cassandra connection
        self._connect_rabbit()

    def _connect_rabbit(self):
        rabbit_server = self._args.rabbit_server
        rabbit_port = self._args.rabbit_port
        rabbit_user = self._args.rabbit_user
        rabbit_password = self._args.rabbit_password
        rabbit_vhost = self._args.rabbit_vhost
        rabbit_ha_mode = self._args.rabbit_ha_mode

        self._db_resync_done = gevent.event.Event()

        q_name = 'svc_mon.%s' % (socket.gethostname())
        self._vnc_kombu = VncKombuClient(rabbit_server, rabbit_port,
                                         rabbit_user, rabbit_password,
                                         rabbit_vhost, rabbit_ha_mode, q_name,
                                         self._vnc_subscribe_callback,
                                         self.logger.log)

        self._cassandra = ServiceMonitorDB(self._args, self.logger)
        DBBaseSM.init(self, self.logger, self._cassandra)

    # end _connect_rabbit

    def _vnc_subscribe_callback(self, oper_info):
        self._db_resync_done.wait()
        try:
            self._vnc_subscribe_actions(oper_info)
        except Exception:
            cgitb_error_log(self)

    def _vnc_subscribe_actions(self, oper_info):
        try:
            msg = "Notification Message: %s" % (pformat(oper_info))
            self.logger.log_debug(msg)
            obj_type = oper_info['type'].replace('-', '_')
            obj_class = DBBaseSM.get_obj_type_map().get(obj_type)
            if obj_class is None:
                return

            if oper_info['oper'] == 'CREATE' or oper_info['oper'] == 'UPDATE':
                dependency_tracker = DependencyTracker(
                    DBBaseSM.get_obj_type_map(), self._REACTION_MAP)
                obj_id = oper_info['uuid']
                obj = obj_class.get(obj_id)
                if obj is not None:
                    dependency_tracker.evaluate(obj_type, obj)
                else:
                    obj = obj_class.locate(obj_id)
                obj.update()
                dependency_tracker.evaluate(obj_type, obj)
            elif oper_info['oper'] == 'DELETE':
                obj_id = oper_info['uuid']
                obj = obj_class.get(obj_id)
                if obj is None:
                    return
                dependency_tracker = DependencyTracker(
                    DBBaseSM.get_obj_type_map(), self._REACTION_MAP)
                dependency_tracker.evaluate(obj_type, obj)
                obj_class.delete(obj_id)
            else:
                # unknown operation
                self.logger.log_error('Unknown operation %s' %
                                      oper_info['oper'])
                return

            if obj is None:
                self.logger.log_error('Error while accessing %s uuid %s' %
                                      (obj_type, obj_id))
                return

        except Exception:
            cgitb_error_log(self)

        for sas_id in dependency_tracker.resources.get('service_appliance_set',
                                                       []):
            sas_obj = ServiceApplianceSetSM.get(sas_id)
            if sas_obj is not None:
                sas_obj.add()

        for lb_pool_id in dependency_tracker.resources.get(
                'loadbalancer_pool', []):
            lb_pool = LoadbalancerPoolSM.get(lb_pool_id)
            if lb_pool is not None:
                lb_pool.add()

        for si_id in dependency_tracker.resources.get('service_instance', []):
            si = ServiceInstanceSM.get(si_id)
            if si:
                si.state = 'launch'
                self._create_service_instance(si)
            else:
                self.logger.log_info("Deleting SI %s" % si_id)
                for vm_id in dependency_tracker.resources.get(
                        'virtual_machine', []):
                    vm = VirtualMachineSM.get(vm_id)
                    self._delete_service_instance(vm)
                self.logger.log_info("SI %s deletion succeed" % si_id)

        for vn_id in dependency_tracker.resources.get('virtual_network', []):
            vn = VirtualNetworkSM.get(vn_id)
            if vn:
                for si_id in ServiceInstanceSM:
                    si = ServiceInstanceSM.get(si_id)
                    intf_list = []
                    if si.params:
                        intf_list = si.params.get('interface_list', [])
                    for intf in intf_list:
                        if (':').join(vn.fq_name) in intf.values():
                            self._create_service_instance(si)

        for vmi_id in dependency_tracker.resources.get(
                'virtual_machine_interface', []):
            vmi = VirtualMachineInterfaceSM.get(vmi_id)
            if vmi:
                for vm_id in dependency_tracker.resources.get(
                        'virtual_machine', []):
                    vm = VirtualMachineSM.get(vm_id)
                    self.port_delete_or_si_link(vm, vmi)
            else:
                for irt_id in dependency_tracker.resources.get(
                        'interface_route_table', []):
                    self._delete_interface_route_table(irt_id)

        for fip_id in dependency_tracker.resources.get('floating_ip', []):
            fip = FloatingIpSM.get(fip_id)
            if fip:
                for vmi_id in fip.virtual_machine_interfaces:
                    vmi = VirtualMachineInterfaceSM.get(vmi_id)
                    if vmi and vmi.virtual_ip:
                        self.netns_manager.add_fip_to_vip_vmi(vmi, fip)

        for lr_id in dependency_tracker.resources.get('logical_router', []):
            lr = LogicalRouterSM.get(lr_id)
            if lr:
                self.snat_agent.update_snat_instance(lr)

    def post_init(self, vnc_lib, args=None):
        # api server
        self._vnc_lib = vnc_lib

        self._nova_client = importutils.import_object(
            'svc_monitor.nova_client.ServiceMonitorNovaClient', self._args,
            self.logger)

        # agent manager
        self._agent_manager = AgentManager()

        # load vrouter scheduler
        self.vrouter_scheduler = importutils.import_object(
            self._args.si_netns_scheduler_driver, self._vnc_lib,
            self._nova_client, self._args)

        # load virtual machine instance manager
        self.vm_manager = importutils.import_object(
            'svc_monitor.virtual_machine_manager.VirtualMachineManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load network namespace instance manager
        self.netns_manager = importutils.import_object(
            'svc_monitor.instance_manager.NetworkNamespaceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load a vrouter instance manager
        self.vrouter_manager = importutils.import_object(
            'svc_monitor.vrouter_instance_manager.VRouterInstanceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load PNF instance manager
        self.ps_manager = importutils.import_object(
            'svc_monitor.physical_service_manager.PhysicalServiceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load a loadbalancer agent
        self.loadbalancer_agent = LoadbalancerAgent(self, self._vnc_lib,
                                                    self._cassandra,
                                                    self._args)
        self._agent_manager.register_agent(self.loadbalancer_agent)

        # load a snat agent
        self.snat_agent = SNATAgent(self, self._vnc_lib, self._cassandra,
                                    self._args)
        self._agent_manager.register_agent(self.snat_agent)

        # load port tuple agent
        self.port_tuple_agent = PortTupleAgent(self, self._vnc_lib,
                                               self._cassandra, self._args,
                                               self.logger)
        self._agent_manager.register_agent(self.port_tuple_agent)

        # Read the cassandra and populate the entry in ServiceMonitor DB
        self.sync_sm()

        # create default analyzer template
        self._create_default_template('analyzer-template',
                                      'analyzer',
                                      flavor='m1.medium',
                                      image_name='analyzer')
        # create default NAT template
        self._create_default_template('nat-template',
                                      'firewall',
                                      svc_mode='in-network-nat',
                                      image_name='analyzer',
                                      flavor='m1.medium')
        # create default netns SNAT template
        self._create_default_template('netns-snat-template',
                                      'source-nat',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        # create default loadbalancer template
        self._create_default_template('haproxy-loadbalancer-template',
                                      'loadbalancer',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        self._create_default_template('docker-template',
                                      'firewall',
                                      svc_mode='transparent',
                                      image_name="ubuntu",
                                      hypervisor_type='vrouter-instance',
                                      vrouter_instance_type='docker',
                                      instance_data={"command": "/bin/bash"})

        # upgrade handling
        self.upgrade()

        # check services
        self.launch_services()

        self._db_resync_done.set()

    def upgrade(self):
        for si in ServiceInstanceSM.values():
            st = ServiceTemplateSM.get(si.service_template)
            if not st:
                continue

            vm_id_list = list(si.virtual_machines)
            for vm_id in vm_id_list:
                vm = VirtualMachineSM.get(vm_id)
                if vm.virtualization_type:
                    continue

                try:
                    nova_vm = self._nova_client.oper('servers',
                                                     'get',
                                                     si.proj_name,
                                                     id=vm_id)
                except nc_exc.NotFound:
                    nova_vm = None

                if nova_vm:
                    vm_name = nova_vm.name
                    vm.proj_fq_name = nova_vm.name.split('__')[0:2]
                else:
                    vm_name = vm.name

                if not vm_name.split('__')[-1].isdigit():
                    continue

                vm.virtualization_type = st.virtualization_type
                self._delete_service_instance(vm)

    def launch_services(self):
        for si in ServiceInstanceSM.values():
            self._create_service_instance(si)

    def sync_sm(self):
        # Read and Sync all DBase
        for cls in DBBaseSM.get_obj_type_map().values():
            for obj in cls.list_obj():
                cls.locate(obj['uuid'], obj)

        # Link SI and VM
        for vm in VirtualMachineSM.values():
            if vm.service_instance:
                continue
            for vmi_id in vm.virtual_machine_interfaces:
                vmi = VirtualMachineInterfaceSM.get(vmi_id)
                if not vmi:
                    continue
                self.port_delete_or_si_link(vm, vmi)

        # invoke port tuple handling
        self.port_tuple_agent.update_port_tuples()

        # Load the loadbalancer driver
        self.loadbalancer_agent.load_drivers()

        # Invoke the loadbalancer pools
        for lb_pool in LoadbalancerPoolSM.values():
            lb_pool.add()

        # Audit the lb pools
        self.loadbalancer_agent.audit_lb_pools()

        # Audit the SNAT instances
        self.snat_agent.audit_snat_instances()

    # end sync_sm

    # create service template
    def _create_default_template(self,
                                 st_name,
                                 svc_type,
                                 svc_mode=None,
                                 hypervisor_type='virtual-machine',
                                 image_name=None,
                                 flavor=None,
                                 scaling=False,
                                 vrouter_instance_type=None,
                                 instance_data=None):
        domain_name = 'default-domain'
        domain_fq_name = [domain_name]
        st_fq_name = [domain_name, st_name]
        self.logger.log_info("Creating %s %s hypervisor %s" %
                             (domain_name, st_name, hypervisor_type))

        domain_obj = None
        for domain in DomainSM.values():
            if domain.fq_name == domain_fq_name:
                domain_obj = Domain()
                domain_obj.uuid = domain.uuid
                domain_obj.fq_name = domain_fq_name
                break
        if not domain_obj:
            self.logger.log_error("%s domain not found" % (domain_name))
            return

        for st in ServiceTemplateSM.values():
            if st.fq_name == st_fq_name:
                self.logger.log_info("%s exists uuid %s" %
                                     (st.name, str(st.uuid)))
                return

        svc_properties = ServiceTemplateType()
        svc_properties.set_service_type(svc_type)
        svc_properties.set_service_mode(svc_mode)
        svc_properties.set_service_virtualization_type(hypervisor_type)
        svc_properties.set_image_name(image_name)
        svc_properties.set_flavor(flavor)
        svc_properties.set_ordered_interfaces(True)
        svc_properties.set_service_scaling(scaling)

        # set interface list
        if svc_type == 'analyzer':
            if_list = [['left', False]]
        elif hypervisor_type == 'network-namespace':
            if_list = [['right', True], ['left', True]]
        else:
            if_list = [['management', False], ['left', False],
                       ['right', False]]

        for itf in if_list:
            if_type = ServiceTemplateInterfaceType(shared_ip=itf[1])
            if_type.set_service_interface_type(itf[0])
            svc_properties.add_interface_type(if_type)

        if vrouter_instance_type is not None:
            svc_properties.set_vrouter_instance_type(vrouter_instance_type)

        if instance_data is not None:
            svc_properties.set_instance_data(
                json.dumps(instance_data, separators=(',', ':')))

        st_obj = ServiceTemplate(name=st_name, domain_obj=domain)
        st_obj.set_service_template_properties(svc_properties)
        try:
            st_uuid = self._vnc_lib.service_template_create(st_obj)
        except Exception as e:
            self.logger.log_error("%s create failed with error %s" %
                                  (st_name, str(e)))
            return

        # Create the service template in local db
        ServiceTemplateSM.locate(st_uuid)

        self.logger.log_info("%s created with uuid %s" %
                             (st_name, str(st_uuid)))

    #_create_default_analyzer_template

    def port_delete_or_si_link(self, vm, vmi):
        if vmi.port_tuple:
            return
        if (vmi.service_instance and vmi.virtual_machine == None):
            self.vm_manager.cleanup_svc_vm_ports([vmi.uuid])
            return

        if not vm or vm.service_instance:
            return
        if not vmi.if_type:
            return

        si_fq_name = vmi.name.split('__')[0:3]
        index = int(vmi.name.split('__')[3]) - 1
        for si in ServiceInstanceSM.values():
            if si.fq_name != si_fq_name:
                continue
            st = ServiceTemplateSM.get(si.service_template)
            self.vm_manager.link_si_to_vm(si, st, index, vm.uuid)
            return

    def _create_service_instance(self, si):
        if si.state == 'active':
            return
        st = ServiceTemplateSM.get(si.service_template)
        if not st:
            self.logger.log_error("template not found for %s" %
                                  ((':').join(si.fq_name)))
            return
        if st.params and st.params.get('version', 1) == 2:
            return

        self.logger.log_info("Creating SI %s (%s)" %
                             ((':').join(si.fq_name), st.virtualization_type))
        try:
            if st.virtualization_type == 'virtual-machine':
                self.vm_manager.create_service(st, si)
            elif st.virtualization_type == 'network-namespace':
                self.netns_manager.create_service(st, si)
            elif st.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.create_service(st, si)
            elif st.virtualization_type == 'physical-device':
                self.ps_manager.create_service(st, si)
            else:
                self.logger.log_error("Unknown virt type: %s" %
                                      st.virtualization_type)
        except Exception:
            cgitb_error_log(self)
        si.launch_count += 1
        self.logger.log_info("SI %s creation success" % (':').join(si.fq_name))

    def _delete_service_instance(self, vm):
        self.logger.log_info("Deleting VM %s %s" %
                             ((':').join(vm.fq_name), vm.uuid))

        try:
            if vm.virtualization_type == svc_info.get_vm_instance_type():
                self.vm_manager.delete_service(vm)
            elif vm.virtualization_type == svc_info.get_netns_instance_type():
                self.netns_manager.delete_service(vm)
            elif vm.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.delete_service(vm)
            elif vm.virtualization_type == 'physical-device':
                self.ps_manager.delete_service(vm)
        except Exception:
            cgitb_error_log(self)

        # generate UVE
        si_fq_name = vm.display_name.split('__')[:-2]
        si_fq_str = (':').join(si_fq_name)
        self.logger.uve_svc_instance(si_fq_str,
                                     status='DELETE',
                                     vms=[{
                                         'uuid': vm.uuid
                                     }])
        return True

    def _relaunch_service_instance(self, si):
        si.state = 'relaunch'
        self._create_service_instance(si)

    def _check_service_running(self, si):
        st = ServiceTemplateSM.get(si.service_template)
        if st.params and st.params.get('version', 1) == 2:
            return
        if st.virtualization_type == 'virtual-machine':
            status = self.vm_manager.check_service(si)
        elif st.virtualization_type == 'network-namespace':
            status = self.netns_manager.check_service(si)
        elif st.virtualization_type == 'vrouter-instance':
            status = self.vrouter_manager.check_service(si)
        elif st.virtualization_type == 'physical-device':
            status = self.ps_manager.check_service(si)
        return status

    def _delete_interface_route_table(self, irt_uuid):
        try:
            self._vnc_lib.interface_route_table_delete(id=irt_uuid)
            InterfaceRouteTableSM.delete(irt_uuid)
        except (NoIdError, RefsExistError):
            return

    def _delete_shared_vn(self, vn_uuid):
        try:
            self.logger.log_info("Deleting vn %s" % (vn_uuid))
            self._vnc_lib.virtual_network_delete(id=vn_uuid)
            VirtualNetworkSM.delete(vn_uuid)
        except (NoIdError, RefsExistError):
            pass

    @staticmethod
    def reset():
        for cls in DBBaseSM.get_obj_type_map().values():
            cls.reset()
class SvcMonitor(object):

    def __init__(self, args=None):
        self._args = args

        # initialize discovery client
        self._disc = None
        if self._args.disc_server_ip and self._args.disc_server_port:
            self._disc = client.DiscoveryClient(
                self._args.disc_server_ip,
                self._args.disc_server_port,
                ModuleNames[Module.SVC_MONITOR])
        # initialize logger
        self.logger = ServiceMonitorLogger(self._disc, args)

        # rotating log file for catchall errors
        self._err_file = self._args.trace_file
        self._svc_err_logger = logging.getLogger('SvcErrLogger')
        self._svc_err_logger.setLevel(logging.ERROR)
        try:
            with open(self._err_file, 'a'):
                handler = logging.handlers.RotatingFileHandler(
                    self._err_file, maxBytes=64*1024, backupCount=2)
                self._svc_err_logger.addHandler(handler)
        except IOError:
            self.logger.warning("Failed to open trace file %s" %
                                    self._err_file)

        # init cassandra
        self._cassandra = ServiceMonitorDB(self._args, self.logger)
        DBBaseSM.init(self, self.logger, self._cassandra)

        # init rabbit connection
        self.rabbit = RabbitConnection(self.logger, args)
        self.rabbit._connect_rabbit()

    def post_init(self, vnc_lib, args=None):
        # api server
        self._vnc_lib = vnc_lib

        self._nova_client = importutils.import_object(
            'svc_monitor.nova_client.ServiceMonitorNovaClient',
            self._args, self.logger)

        # agent manager
        self._agent_manager = AgentManager()

        # load vrouter scheduler
        self.vrouter_scheduler = importutils.import_object(
            self._args.si_netns_scheduler_driver,
            self._vnc_lib, self._nova_client,
            self._disc, self.logger, self._args)

        # load virtual machine instance manager
        self.vm_manager = importutils.import_object(
            'svc_monitor.virtual_machine_manager.VirtualMachineManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load network namespace instance manager
        self.netns_manager = importutils.import_object(
            'svc_monitor.instance_manager.NetworkNamespaceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client, self._agent_manager,
            self._args)

        # load a vrouter instance manager
        self.vrouter_manager = importutils.import_object(
            'svc_monitor.vrouter_instance_manager.VRouterInstanceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client,
            self._agent_manager, self._args)

        # load PNF instance manager
        self.ps_manager = importutils.import_object(
            'svc_monitor.physical_service_manager.PhysicalServiceManager',
            self._vnc_lib, self._cassandra, self.logger,
            self.vrouter_scheduler, self._nova_client,
            self._agent_manager, self._args)

        # load a loadbalancer agent
        self.loadbalancer_agent = LoadbalancerAgent(
            self, self._vnc_lib,
            self._cassandra, self._args)
        self._agent_manager.register_agent(self.loadbalancer_agent)

        # load a snat agent
        self.snat_agent = SNATAgent(self, self._vnc_lib,
                                    self._cassandra, self._args)
        self._agent_manager.register_agent(self.snat_agent)

        # load port tuple agent
        self.port_tuple_agent = PortTupleAgent(self, self._vnc_lib,
            self._cassandra, self._args, self.logger)
        self._agent_manager.register_agent(self.port_tuple_agent)

        # Read the cassandra and populate the entry in ServiceMonitor DB
        self.sync_sm()

        # create default analyzer template
        self._create_default_template('analyzer-template', 'analyzer',
                                      flavor='m1.medium',
                                      image_name='analyzer')
        # create default NAT template
        self._create_default_template('nat-template', 'firewall',
                                      svc_mode='in-network-nat',
                                      image_name='analyzer',
                                      flavor='m1.medium')
        # create default netns SNAT template
        self._create_default_template('netns-snat-template', 'source-nat',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        # create default loadbalancer template
        self._create_default_template('haproxy-loadbalancer-template',
                                      'loadbalancer',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        self._create_default_template('docker-template', 'firewall',
                                      svc_mode='transparent',
                                      image_name="ubuntu",
                                      hypervisor_type='vrouter-instance',
                                      vrouter_instance_type='docker',
                                      instance_data={
                                          "command": "/bin/bash"
                                      })

        # upgrade handling
        self.upgrade()

        # check services
        self.vrouter_scheduler.vrouters_running()
        self.launch_services()

        self.rabbit._db_resync_done.set()

    def _upgrade_instance_ip(self, vm):
        for vmi_id in vm.virtual_machine_interfaces:
            vmi = VirtualMachineInterfaceSM.get(vmi_id)
            if not vmi:
                continue

            for iip_id in vmi.instance_ips:
                iip = InstanceIpSM.get(iip_id)
                if not iip or iip.service_instance_ip:
                    continue
                iip_obj = InstanceIp()
                iip_obj.name = iip.name
                iip_obj.uuid = iip.uuid
                iip_obj.set_service_instance_ip(True)
                try:
                    self._vnc_lib.instance_ip_update(iip_obj)
                except NoIdError:
                    self.logger.error("upgrade instance ip to service ip failed %s" % (iip.name))
                    continue

    def upgrade(self):
        for si in ServiceInstanceSM.values():
            st = ServiceTemplateSM.get(si.service_template)
            if not st:
                continue

            vm_id_list = list(si.virtual_machines)
            for vm_id in vm_id_list:
                vm = VirtualMachineSM.get(vm_id)
                self._upgrade_instance_ip(vm)
                if vm.virtualization_type:
                    continue

                try:
                    nova_vm = self._nova_client.oper('servers', 'get',
                        si.proj_name, id=vm_id)
                except nc_exc.NotFound:
                    nova_vm = None

                if nova_vm:
                    vm_name = nova_vm.name
                    vm.proj_fq_name = nova_vm.name.split('__')[0:2]
                else:
                    vm_name = vm.name

                if not vm_name.split('__')[-1].isdigit():
                    continue

                vm.virtualization_type = st.virtualization_type
                self.delete_service_instance(vm)

    def launch_services(self):
        for si in ServiceInstanceSM.values():
            self.create_service_instance(si)

    def sync_sm(self):
        # Read and Sync all DBase
        for cls in DBBaseSM.get_obj_type_map().values():
            for obj in cls.list_obj():
                cls.locate(obj['uuid'], obj)

        # Link SI and VM
        for vm in VirtualMachineSM.values():
            if vm.service_instance:
                continue
            for vmi_id in vm.virtual_machine_interfaces:
                vmi = VirtualMachineInterfaceSM.get(vmi_id)
                if not vmi:
                    continue
                self.port_delete_or_si_link(vm, vmi)

        # invoke port tuple handling
        try:
            self.port_tuple_agent.update_port_tuples()
        except Exception:
            cgitb_error_log(self)

        # Load the loadbalancer driver
        self.loadbalancer_agent.load_drivers()

        # Invoke the loadbalancer pools
        for lb_pool in LoadbalancerPoolSM.values():
            lb_pool.add()

        # Audit the lb pools
        self.loadbalancer_agent.audit_lb_pools()

        # Audit the SNAT instances
        self.snat_agent.audit_snat_instances()
    # end sync_sm

    # create service template
    def _create_default_template(self, st_name, svc_type, svc_mode=None,
                                 hypervisor_type='virtual-machine',
                                 image_name=None, flavor=None, scaling=False,
                                 vrouter_instance_type=None,
                                 instance_data=None):
        domain_name = 'default-domain'
        domain_fq_name = [domain_name]
        st_fq_name = [domain_name, st_name]
        self.logger.info("Creating %s %s hypervisor %s" %
                             (domain_name, st_name, hypervisor_type))

        domain_obj = None
        for domain in DomainSM.values():
            if domain.fq_name == domain_fq_name:
                domain_obj = Domain()
                domain_obj.uuid = domain.uuid
                domain_obj.fq_name = domain_fq_name
                break
        if not domain_obj:
            self.logger.error("%s domain not found" % (domain_name))
            return

        for st in ServiceTemplateSM.values():
            if st.fq_name == st_fq_name:
                self.logger.info("%s exists uuid %s" %
                                     (st.name, str(st.uuid)))
                return

        svc_properties = ServiceTemplateType()
        svc_properties.set_service_type(svc_type)
        svc_properties.set_service_mode(svc_mode)
        svc_properties.set_service_virtualization_type(hypervisor_type)
        svc_properties.set_image_name(image_name)
        svc_properties.set_flavor(flavor)
        svc_properties.set_ordered_interfaces(True)
        svc_properties.set_service_scaling(scaling)

        # set interface list
        if svc_type == 'analyzer':
            if_list = [['left', False]]
        elif hypervisor_type == 'network-namespace':
            if_list = [['right', True], ['left', True]]
        else:
            if_list = [
                ['management', False], ['left', False], ['right', False]]

        for itf in if_list:
            if_type = ServiceTemplateInterfaceType(shared_ip=itf[1])
            if_type.set_service_interface_type(itf[0])
            svc_properties.add_interface_type(if_type)

        if vrouter_instance_type is not None:
            svc_properties.set_vrouter_instance_type(vrouter_instance_type)

        if instance_data is not None:
            svc_properties.set_instance_data(
                json.dumps(instance_data, separators=(',', ':')))

        st_obj = ServiceTemplate(name=st_name, domain_obj=domain)
        st_obj.set_service_template_properties(svc_properties)
        try:
            st_uuid = self._vnc_lib.service_template_create(st_obj)
        except Exception as e:
            self.logger.error("%s create failed with error %s" %
                                  (st_name, str(e)))
            return

        # Create the service template in local db
        ServiceTemplateSM.locate(st_uuid)

        self.logger.info("%s created with uuid %s" %
                             (st_name, str(st_uuid)))
    #_create_default_analyzer_template

    def port_delete_or_si_link(self, vm, vmi):
        if vmi.port_tuple:
            return
        if (vmi.service_instance and vmi.virtual_machine == None):
            self.vm_manager.cleanup_svc_vm_ports([vmi.uuid])
            return

        if not vm or vm.service_instance:
            return
        if not vmi.if_type:
            return

        if len(vmi.name.split('__')) < 4:
            return
        si_fq_name = vmi.name.split('__')[0:3]
        index = int(vmi.name.split('__')[3]) - 1
        for si in ServiceInstanceSM.values():
            if si.fq_name != si_fq_name:
                continue
            st = ServiceTemplateSM.get(si.service_template)
            self.vm_manager.link_si_to_vm(si, st, index, vm.uuid)
            return

    def create_service_instance(self, si):
        if si.state == 'active':
            return
        st = ServiceTemplateSM.get(si.service_template)
        if not st:
            self.logger.error("template not found for %s" %
                                  ((':').join(si.fq_name)))
            return
        if st.params and st.params.get('version', 1) == 2:
            return

        self.logger.info("Creating SI %s (%s)" %
                             ((':').join(si.fq_name), st.virtualization_type))
        try:
            if st.virtualization_type == 'virtual-machine':
                self.vm_manager.create_service(st, si)
            elif st.virtualization_type == 'network-namespace':
                self.netns_manager.create_service(st, si)
            elif st.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.create_service(st, si)
            elif st.virtualization_type == 'physical-device':
                self.ps_manager.create_service(st, si)
            else:
                self.logger.error("Unknown virt type: %s" %
                                      st.virtualization_type)
        except Exception:
            cgitb_error_log(self)
        si.launch_count += 1
        self.logger.info("SI %s creation success" % (':').join(si.fq_name))

    def delete_service_instance(self, vm):
        self.logger.info("Deleting VM %s %s for SI %s" %
            ((':').join(vm.fq_name), vm.uuid, vm.service_id))

        try:
            if vm.virtualization_type == svc_info.get_vm_instance_type():
                self.vm_manager.delete_service(vm)
            elif vm.virtualization_type == svc_info.get_netns_instance_type():
                self.netns_manager.delete_service(vm)
            elif vm.virtualization_type == 'vrouter-instance':
                self.vrouter_manager.delete_service(vm)
            elif vm.virtualization_type == 'physical-device':
                self.ps_manager.delete_service(vm)
            self.logger.info("Deleted VM %s %s for SI %s" %
                ((':').join(vm.fq_name), vm.uuid, vm.service_id))
        except Exception:
            cgitb_error_log(self)

        # generate UVE
        si_fq_name = vm.display_name.split('__')[:-2]
        si_fq_str = (':').join(si_fq_name)
        self.logger.uve_svc_instance(si_fq_str, status='DELETE',
                                     vms=[{'uuid': vm.uuid}])
        return True

    def _relaunch_service_instance(self, si):
        si.state = 'relaunch'
        self.create_service_instance(si)

    def _check_service_running(self, si):
        st = ServiceTemplateSM.get(si.service_template)
        if st.params and st.params.get('version', 1) == 2:
            return
        if st.virtualization_type == 'virtual-machine':
            status = self.vm_manager.check_service(si)
        elif st.virtualization_type == 'network-namespace':
            status = self.netns_manager.check_service(si)
        elif st.virtualization_type == 'vrouter-instance':
            status = self.vrouter_manager.check_service(si)
        elif st.virtualization_type == 'physical-device':
            status = self.ps_manager.check_service(si)
        return status

    def delete_interface_route_table(self, irt_uuid):
        try:
            self._vnc_lib.interface_route_table_delete(id=irt_uuid)
            InterfaceRouteTableSM.delete(irt_uuid)
        except (NoIdError, RefsExistError):
            return

    def _delete_shared_vn(self, vn_uuid):
        try:
            self.logger.info("Deleting vn %s" % (vn_uuid))
            self._vnc_lib.virtual_network_delete(id=vn_uuid)
            VirtualNetworkSM.delete(vn_uuid)
        except (NoIdError, RefsExistError):
            pass

    @staticmethod
    def reset():
        for cls in DBBaseSM.get_obj_type_map().values():
            cls.reset()