Exemplo n.º 1
0
 def __init__(self, config):
     self._config = config
     if 'host_ip' in self._config._args:
         host_ip = self._config._args.host_ip
     else:
         host_ip = socket.gethostbyname(socket.getfqdn())
     self._hostname = socket.getfqdn(host_ip)
     self.analytic_api = AnalyticApiClient(self._config)
     self._config.random_collectors = self._config.collectors()
     self._chksum = ""
     if self._config.collectors():
         self._chksum = hashlib.md5("".join(
             self._config.collectors())).hexdigest()
         self._config.random_collectors = random.sample(self._config.collectors(), \
                                                        len(self._config.collectors()))
     self.uve = LinkUve(self._config)
     self._sandesh = self.uve.sandesh_instance()
     self._logger = self.uve.logger()
     self.sleep_time()
     self._sem = Semaphore()
     self._members = None
     self._partitions = None
     self._prouters = {}
     self._vrouter_l2ifs = {}
     self._old_vrouter_l2ifs = {}
     self._config_handler = TopologyConfigHandler(
         self._sandesh, self._config.rabbitmq_params(),
         self._config.cassandra_params(), host_ip)
     self.constnt_schdlr = ConsistentScheduler(
         self.uve._moduleid,
         zookeeper=self._config.zookeeper_server(),
         delete_hndlr=self._del_uves,
         logger=self._logger,
         cluster_id=self._config.cluster_id())
Exemplo n.º 2
0
    def run(self):
        """ @sighup
        SIGHUP handler to indicate configuration changes 
        """
        gevent.signal(signal.SIGHUP, self.sighup_handler)

        self._sem = Semaphore()
        self.constnt_schdlr = ConsistentScheduler(
            self.uve._moduleid,
            zookeeper=self._config.zookeeper_server(),
            delete_hndlr=self._del_uves,
            cluster_id=self._config.cluster_id())

        while self._keep_running:
            self.scan_data()
            if self.constnt_schdlr.schedule(self.prouters):
                try:
                    with self._sem:
                        self.compute()
                        self.send_uve()
                except Exception as e:
                    traceback.print_exc()
                    print str(e)
                gevent.sleep(self._sleep_time)
            else:
                gevent.sleep(1)
        self.constnt_schdlr.finish()
Exemplo n.º 3
0
 def __init__(self, config):
     self._config = config
     self._config.random_collectors = self._config.collectors()
     self._chksum = ""
     if self._config.collectors():
          self._chksum = hashlib.md5("".join(self._config.collectors())).hexdigest()
          self._config.random_collectors = random.sample(self._config.collectors(), \
                                                         len(self._config.collectors()))
     self.uve = SnmpUve(self._config)
     self._sandesh = self.uve.sandesh_instance()
     self._hostname = socket.gethostname()
     self._logger = self.uve.logger()
     self.sleep_time()
     self.last = set()
     self._sem = Semaphore()
     self._config.set_cb(self.notify)
     self._mnt = MaxNinTtime(3, self._sleep_time)
     self._state = 'full_scan' # replace it w/ fsm
     self._if_data = None # replace it w/ fsm
     self._cleanup = None
     self._members = None
     self._partitions = None
     self._prouters = {}
     self._config_handler = SnmpConfigHandler(self._sandesh,
         self._config.rabbitmq_params(), self._config.cassandra_params())
     self._consistent_scheduler = ConsistentScheduler(self._config._name,
         zookeeper=self._config.zookeeper_server(),
         delete_hndlr=self._del_uves, logger=self._logger,
         cluster_id=self._config.cluster_id())
Exemplo n.º 4
0
    def run(self):
        """ @sighup
        SIGHUP handler to indicate configuration changes 
        """
        gevent.signal(signal.SIGHUP, self.sighup_handler)

        self._sem = Semaphore()
        self.constnt_schdlr = ConsistentScheduler(
            self.uve._moduleid,
            zookeeper=self._config.zookeeper_server(),
            delete_hndlr=self._del_uves,
            logger=self._logger,
            cluster_id=self._config.cluster_id())

        while self._keep_running:
            if not self._vnc:
                self._vnc = self._config.vnc_api()
            self.scan_data()
            if self.constnt_schdlr.schedule(self.prouters):
                members = self.constnt_schdlr.members()
                partitions = self.constnt_schdlr.partitions()
                prouters = map(lambda x: x.name,
                               self.constnt_schdlr.work_items())
                self._send_topology_uve(members, partitions, prouters)
                try:
                    with self._sem:
                        self.compute()
                        self.send_uve()
                except Exception as e:
                    traceback.print_exc()
                    print str(e)
                gevent.sleep(self._sleep_time)
            else:
                gevent.sleep(1)
        self.constnt_schdlr.finish()
Exemplo n.º 5
0
 def run(self):
     i = 0
     self._sem = Semaphore()
     self._logger.debug('Starting.. %s' % str(
                 self._config.zookeeper_server()))
     constnt_schdlr = ConsistentScheduler(
                         self._config._name,
                         zookeeper=self._config.zookeeper_server(),
                         delete_hndlr=self._del_uves,
                         logger=self._logger, 
                         cluster_id=self._config.cluster_id())
     while self._keep_running:
         self._logger.debug('@run: ittr(%d)' % i)
         if constnt_schdlr.schedule(self._config.devices()):
             members = constnt_schdlr.members()
             partitions = constnt_schdlr.partitions()
             prouters = map(lambda x: x.name, constnt_schdlr.work_items())
             self._send_snmp_collector_uve(members, partitions, prouters)
             sleep_time = self.do_work(i, constnt_schdlr.work_items())
             self._logger.debug('done work %s' % str(prouters))
             i += 1
             gevent.sleep(sleep_time)
         else:
             gevent.sleep(1)
     constnt_schdlr.finish()
Exemplo n.º 6
0
    def run(self):
        """ @sighup
        SIGHUP handler to indicate configuration changes
        """
        gevent.signal(signal.SIGHUP, self.sighup_handler)

        i = 0
        self._sem = Semaphore()
        self._logger.debug('Starting.. %s' %
                           str(self._config.zookeeper_server()))
        constnt_schdlr = ConsistentScheduler(
            self._config._name,
            zookeeper=self._config.zookeeper_server(),
            delete_hndlr=self._del_uves,
            logger=self._logger,
            cluster_id=self._config.cluster_id())
        while self._keep_running:
            self._logger.debug('@run: ittr(%d)' % i)
            if constnt_schdlr.schedule(self._config.devices()):
                members = constnt_schdlr.members()
                partitions = constnt_schdlr.partitions()
                self._send_snmp_collector_uve(members, partitions,
                                              constnt_schdlr.work_items())
                sleep_time = self.do_work(i, constnt_schdlr.work_items())
                self._logger.debug('done work %s' % str(self._prouters.keys()))
                i += 1
                gevent.sleep(sleep_time)
            else:
                gevent.sleep(1)
        constnt_schdlr.finish()
Exemplo n.º 7
0
    def run(self):
       
        """ @sighup
        SIGHUP handler to indicate configuration changes
        """
        gevent.signal(signal.SIGHUP, self.sighup_handler) 

        i = 0
        self._sem = Semaphore()
        self._logger.debug('Starting.. %s' % str(
                    self._config.zookeeper_server()))
        constnt_schdlr = ConsistentScheduler(
                            self._config._name,
                            zookeeper=self._config.zookeeper_server(),
                            delete_hndlr=self._del_uves,
                            logger=self._logger, 
                            cluster_id=self._config.cluster_id())
        while self._keep_running:
            self._logger.debug('@run: ittr(%d)' % i)
            if constnt_schdlr.schedule(self._config.devices()):
                members = constnt_schdlr.members()
                partitions = constnt_schdlr.partitions()
                self._send_snmp_collector_uve(members, partitions,
                    constnt_schdlr.work_items())
                sleep_time = self.do_work(i, constnt_schdlr.work_items())
                self._logger.debug('done work %s' % str(self._prouters.keys()))
                i += 1
                gevent.sleep(sleep_time)
            else:
                gevent.sleep(1)
        constnt_schdlr.finish()
Exemplo n.º 8
0
    def run(self):

        """ @sighup
        SIGHUP handler to indicate configuration changes 
        """
        gevent.signal(signal.SIGHUP, self.sighup_handler)

        self._sem = Semaphore()
        self.constnt_schdlr = ConsistentScheduler(
                            self.uve._moduleid,
                            zookeeper=self._config.zookeeper_server(),
                            delete_hndlr=self._del_uves,
                            logger=self._logger,
                            cluster_id=self._config.cluster_id())

        while self._keep_running:
            if not self._vnc:
                self._vnc = self._config.vnc_api()
            self.scan_data()
            if self.constnt_schdlr.schedule(self.prouters):
                members = self.constnt_schdlr.members()
                partitions = self.constnt_schdlr.partitions()
                self._send_topology_uve(members, partitions,
                    self.constnt_schdlr.work_items())
                try:
                    with self._sem:
                        self.compute()
                        self.send_uve()
                except Exception as e:
                    traceback.print_exc()
                    print str(e)
                gevent.sleep(self._sleep_time)
            else:
                gevent.sleep(1)
        self.constnt_schdlr.finish()
Exemplo n.º 9
0
 def run(self):
     self._sem = Semaphore()
     self.constnt_schdlr = ConsistentScheduler(
                         self.uve._moduleid,
                         zookeeper=self._config.zookeeper_server(),
                         delete_hndlr=self._del_uves)
     while self._keep_running:
         self.scan_data()
         if self.constnt_schdlr.schedule(self.prouters):
             try:
                 with self._sem:
                     self.compute()
                     self.send_uve()
             except Exception as e:
                 import traceback; traceback.print_exc()
                 print str(e)
             gevent.sleep(self._sleep_time)
         else:
             gevent.sleep(1)
     self.constnt_schdlr.finish()
Exemplo n.º 10
0
 def run(self):
     self._sem = Semaphore()
     self.constnt_schdlr = ConsistentScheduler(
                         self.uve._moduleid,
                         zookeeper=self._config.zookeeper_server(),
                         delete_hndlr=self._del_uves)
     while self._keep_running:
         self.scan_data()
         if self.constnt_schdlr.schedule(self.prouters):
             try:
                 with self._sem:
                     self.compute()
                     self.send_uve()
             except Exception as e:
                 import traceback; traceback.print_exc()
                 print str(e)
             gevent.sleep(self._sleep_time)
         else:
             gevent.sleep(1)
     self.constnt_schdlr.finish()
Exemplo n.º 11
0
 def run(self):
     i = 0
     self._sem = Semaphore()
     constnt_schdlr = ConsistentScheduler(
                         self._config._name,
                         zookeeper=self._config.zookeeper_server(),
                         delete_hndlr=self._del_uves,
                         logger=self._logger)
     while self._keep_running:
         self._logger.debug('@run: ittr(%d)' % i)
         if constnt_schdlr.schedule(self._config.devices()):
             sleep_time = self.do_work(i, constnt_schdlr.work_items())
             self._logger.debug('done work %s' % str(
                         map(lambda x: x.name,
                         constnt_schdlr.work_items())))
             i += 1
             gevent.sleep(sleep_time)
         else:
             gevent.sleep(1)
     constnt_schdlr.finish()
Exemplo n.º 12
0
 def run(self):
     i = 0
     self._sem = Semaphore()
     constnt_schdlr = ConsistentScheduler(
                         self._config._name,
                         zookeeper=self._config.zookeeper_server(),
                         delete_hndlr=self._del_uves,
                         logger=self._logger)
     while self._keep_running:
         self._logger.debug('@run: ittr(%d)' % i)
         if constnt_schdlr.schedule(self._config.devices()):
             sleep_time = self.do_work(i, constnt_schdlr.work_items())
             self._logger.debug('done work %s' % str(
                         map(lambda x: x.name,
                         constnt_schdlr.work_items())))
             i += 1
             gevent.sleep(sleep_time)
         else:
             gevent.sleep(1)
     constnt_schdlr.finish()
Exemplo n.º 13
0
class Controller(object):
    def __init__(self, config):
        self._config = config
        self._hostname = socket.gethostname()
        self.analytic_api = AnalyticApiClient(self._config)
        self._config.random_collectors = self._config.collectors()
        self._chksum = ""
        self._api_server_checksum = ""
        if self._config.collectors():
            self._chksum = hashlib.md5("".join(
                self._config.collectors())).hexdigest()
            self._config.random_collectors = random.sample(self._config.collectors(), \
                                                           len(self._config.collectors()))
        if self._config.api_server_list():
            self._api_server_checksum = hashlib.md5("".join(
                self._config.api_server_list())).hexdigest()
            random_api_servers = random.sample(
                self._config.api_server_list(),
                len(self._config.api_server_list()))
            self._config.set_api_server_list(random_api_servers)
        self.uve = LinkUve(self._config)
        self._logger = self.uve.logger()
        self.sleep_time()
        self._keep_running = True
        self._vnc = None
        self._members = None
        self._partitions = None
        self._prouters = None

    def stop(self):
        self._keep_running = False

    def sleep_time(self, newtime=None):
        if newtime:
            self._sleep_time = newtime
        else:
            self._sleep_time = self._config.frequency()
        return self._sleep_time

    def get_vrouters(self):
        self.analytic_api.get_vrouters(True)
        self.vrouters = {}
        self.vrouter_ips = {}
        self.vrouter_macs = {}
        for vr in self.analytic_api.list_vrouters():
            try:
                d = self.analytic_api.get_vrouter(vr, 'VrouterAgent:phy_if')
            except Exception as e:
                traceback.print_exc()
                print str(e)
                d = {}
            if 'VrouterAgent' not in d:
                d['VrouterAgent'] = {}
            try:
                _ipl = self.analytic_api.get_vrouter(
                    vr, 'VrouterAgent:self_ip_list')
            except Exception as e:
                traceback.print_exc()
                print str(e)
                _ipl = {}
            if 'VrouterAgent' in _ipl:
                d['VrouterAgent'].update(_ipl['VrouterAgent'])
            if 'VrouterAgent' not in d or\
                'self_ip_list' not in d['VrouterAgent'] or\
                'phy_if' not in d['VrouterAgent']:
                continue
            self.vrouters[vr] = {
                'ips': d['VrouterAgent']['self_ip_list'],
                'if': d['VrouterAgent']['phy_if'],
            }
            for ip in d['VrouterAgent']['self_ip_list']:
                self.vrouter_ips[ip] = vr  # index
            for intf in d['VrouterAgent']['phy_if']:
                try:
                    self.vrouter_macs[intf['mac_address']] = {}
                    self.vrouter_macs[intf['mac_address']]['vrname'] = vr
                    self.vrouter_macs[
                        intf['mac_address']]['ifname'] = intf['name']
                except:
                    continue

    def get_prouters(self):
        self.analytic_api.get_prouters(True)
        self.prouters = []
        for pr in self.analytic_api.list_prouters():
            try:
                self.prouters.append(
                    PRouter(pr,
                            self.analytic_api.get_prouter(pr, 'PRouterEntry')))
            except Exception as e:
                traceback.print_exc()
                print str(e)

    def _is_linkup(self, prouter, ifindex):
        if 'PRouterEntry' in prouter.data and \
            'ifIndexOperStatusTable' in prouter.data['PRouterEntry']:
            status = filter(
                lambda x: x['ifIndex'] == ifindex,
                prouter.data['PRouterEntry']['ifIndexOperStatusTable'])
            if status and status[0]['ifOperStatus'] == 1:
                return True
        return False

    def _add_link(self, prouter, remote_system_name, local_interface_name,
                  remote_interface_name, local_interface_index,
                  remote_interface_index, link_type):
        d = dict(remote_system_name=remote_system_name,
                 local_interface_name=local_interface_name,
                 remote_interface_name=remote_interface_name,
                 local_interface_index=local_interface_index,
                 remote_interface_index=remote_interface_index,
                 type=link_type)
        if self._is_linkup(prouter, local_interface_index):
            if prouter.name in self.link:
                self.link[prouter.name].append(d)
            else:
                self.link[prouter.name] = [d]
            return True
        return False

    def _chk_lnk(self, pre, index):
        if 'ifIndexOperStatusTable' in pre:
            for d in pre['ifIndexOperStatusTable']:
                if d['ifIndex'] == index:
                    return d['ifOperStatus'] == 1
        return False

    def _send_topology_uve(self, members, partitions, prouters):
        topology_info = TopologyInfo()
        if self._members != members:
            self._members = members
            topology_info.members = members
        if self._partitions != partitions:
            self._partitions = partitions
            topology_info.partitions = partitions
        if self._prouters != prouters:
            self._prouters = prouters
            topology_info.prouters = prouters
        if topology_info != TopologyInfo():
            topology_info.name = self._hostname
            TopologyUVE(data=topology_info).send()

    # end _send_topology_uve

    def bms_links(self, prouter, ifm):
        if self._vnc:
            try:
                for li in self._vnc.logical_interfaces_list(
                )['logical-interfaces']:
                    if prouter.name in li['fq_name']:
                        lif = self._vnc.logical_interface_read(id=li['uuid'])
                        for vmif in lif.get_virtual_machine_interface_refs():
                            vmi = self._vnc.virtual_machine_interface_read(
                                id=vmif['uuid'])
                            for mc in vmi.virtual_machine_interface_mac_addresses.get_mac_address(
                            ):
                                ifi = [
                                    k for k in ifm if ifm[k] in li['fq_name']
                                ][0]
                                rsys = '-'.join(['bms', 'host'] +
                                                mc.split(':'))
                                if self._add_link(
                                        prouter=prouter,
                                        remote_system_name=rsys,
                                        local_interface_name=li['fq_name'][-1],
                                        remote_interface_name='em0',  #no idea
                                        local_interface_index=ifi,
                                        remote_interface_index=
                                        1,  #dont know TODO:FIX
                                        link_type=2):
                                    pass
            except:
                traceback.print_exc()
                self._vnc = None  # refresh

    def compute(self):
        self.link = {}
        for prouter in self.constnt_schdlr.work_items():
            pr, d = prouter.name, prouter.data
            if 'PRouterEntry' not in d or 'ifTable' not in d['PRouterEntry']:
                continue
            self.link[pr] = []
            lldp_ints = []
            ifm = dict(
                map(lambda x: (x['ifIndex'], x['ifDescr']),
                    d['PRouterEntry']['ifTable']))
            self.bms_links(prouter, ifm)
            for pl in d['PRouterEntry']['lldpTable']['lldpRemoteSystemsData']:
                if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                        'lldpLocSysDesc'].startswith('Cisco'):
                    loc_pname = [
                        x for x in d['PRouterEntry']['lldpTable']
                        ['lldpLocalSystemData']['lldpLocPortTable']
                        if x['lldpLocPortNum'] == pl['lldpRemLocalPortNum']
                    ][0]['lldpLocPortDesc']
                    pl['lldpRemLocalPortNum'] = [
                        k for k in ifm if ifm[k] == loc_pname
                    ][0]
                elif d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                        'lldpLocSysDesc'].startswith('Arista'):
                    loc_pname = [
                        x for x in d['PRouterEntry']['lldpTable']
                        ['lldpLocalSystemData']['lldpLocPortTable']
                        if x['lldpLocPortNum'] == pl['lldpRemLocalPortNum']
                    ][0]['lldpLocPortId']
                    pl['lldpRemLocalPortNum'] = [
                        k for k in ifm if ifm[k] == loc_pname
                    ][0]
                if pl['lldpRemLocalPortNum'] in ifm and self._chk_lnk(
                        d['PRouterEntry'], pl['lldpRemLocalPortNum']):
                    if pl['lldpRemPortId'].isdigit():
                        rii = int(pl['lldpRemPortId'])
                    else:
                        try:
                            if d['PRouterEntry']['lldpTable'][
                                    'lldpLocalSystemData'][
                                        'lldpLocSysDesc'].startswith('Arista'):
                                rpn = filter(
                                    lambda y: y['lldpLocPortId'] == pl[
                                        'lldpRemPortId'], [
                                            x for x in self.prouters
                                            if x.name == pl['lldpRemSysName']
                                        ][0].data['PRouterEntry']['lldpTable']
                                    ['lldpLocalSystemData']
                                    ['lldpLocPortTable'])[0]['lldpLocPortId']
                            else:
                                rpn = filter(
                                    lambda y: y['lldpLocPortId'] == pl[
                                        'lldpRemPortId'], [
                                            x for x in self.prouters
                                            if x.name == pl['lldpRemSysName']
                                        ][0].data['PRouterEntry']['lldpTable']
                                    ['lldpLocalSystemData']
                                    ['lldpLocPortTable'])[0]['lldpLocPortDesc']
                            rii = filter(lambda y: y['ifDescr'] == rpn,
                                    [ x for x in self.prouters \
                                    if x.name == pl['lldpRemSysName']][0].data[
                                    'PRouterEntry']['ifTable'])[0]['ifIndex']
                        except:
                            rii = 0

                    if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                            'lldpLocSysDesc'].startswith('Arista'):
                        if self._add_link(
                                prouter=prouter,
                                remote_system_name=pl['lldpRemSysName'],
                                local_interface_name=ifm[
                                    pl['lldpRemLocalPortNum']],
                                remote_interface_name=pl['lldpRemPortId'],
                                local_interface_index=pl[
                                    'lldpRemLocalPortNum'],
                                remote_interface_index=rii,
                                link_type=1):
                            lldp_ints.append(ifm[pl['lldpRemLocalPortNum']])
                    else:
                        if self._add_link(
                                prouter=prouter,
                                remote_system_name=pl['lldpRemSysName'],
                                local_interface_name=ifm[
                                    pl['lldpRemLocalPortNum']],
                                remote_interface_name=pl['lldpRemPortDesc'],
                                local_interface_index=pl[
                                    'lldpRemLocalPortNum'],
                                remote_interface_index=rii,
                                link_type=1):
                            lldp_ints.append(ifm[pl['lldpRemLocalPortNum']])

            vrouter_neighbors = []
            if 'fdbPortIfIndexTable' in d['PRouterEntry']:
                dot1d2snmp = map(
                    lambda x: (x['dot1dBasePortIfIndex'], x['snmpIfIndex']),
                    d['PRouterEntry']['fdbPortIfIndexTable'])
                dot1d2snmp_dict = dict(dot1d2snmp)
                if 'fdbPortTable' in d['PRouterEntry']:
                    for mac_entry in d['PRouterEntry']['fdbPortTable']:
                        if mac_entry['mac'] in self.vrouter_macs:
                            vrouter_mac_entry = self.vrouter_macs[
                                mac_entry['mac']]
                            fdbport = mac_entry['dot1dBasePortIfIndex']
                            try:
                                snmpport = dot1d2snmp_dict[fdbport]
                                ifname = ifm[snmpport]
                            except:
                                continue
                            is_lldp_int = any(ifname == lldp_int
                                              for lldp_int in lldp_ints)
                            if is_lldp_int:
                                continue
                            if self._add_link(
                                    prouter=prouter,
                                    remote_system_name=vrouter_mac_entry[
                                        'vrname'],
                                    local_interface_name=ifname,
                                    remote_interface_name=vrouter_mac_entry[
                                        'ifname'],
                                    local_interface_index=snmpport,
                                    remote_interface_index=
                                    1,  #dont know TODO:FIX
                                    link_type=2):
                                vrouter_neighbors.append(
                                    vrouter_mac_entry['vrname'])
            for arp in d['PRouterEntry']['arpTable']:
                if arp['ip'] in self.vrouter_ips:
                    if arp['mac'] in map(
                            lambda x: x['mac_address'],
                            self.vrouters[self.vrouter_ips[arp['ip']]]['if']):
                        vr_name = arp['ip']
                        vr = self.vrouters[self.vrouter_ips[vr_name]]
                        if self.vrouter_ips[vr_name] in vrouter_neighbors:
                            continue
                        if ifm[arp['localIfIndex']].startswith('vlan'):
                            continue
                        if ifm[arp['localIfIndex']].startswith('irb'):
                            continue
                        is_lldp_int = any(ifm[arp['localIfIndex']] == lldp_int
                                          for lldp_int in lldp_ints)
                        if is_lldp_int:
                            continue
                        if self._add_link(
                                prouter=prouter,
                                remote_system_name=self.vrouter_ips[vr_name],
                                local_interface_name=ifm[arp['localIfIndex']],
                                remote_interface_name=vr['if'][-1]
                            ['name'],  #TODO
                                local_interface_index=arp['localIfIndex'],
                                remote_interface_index=1,  #dont know TODO:FIX
                                link_type=2):
                            pass

    def send_uve(self):
        self.uve.send(self.link)

    def switcher(self):
        gevent.sleep(0)

    def scan_data(self):
        t = []
        t.append(gevent.spawn(self.get_vrouters))
        t.append(gevent.spawn(self.get_prouters))
        gevent.joinall(t)

    def _del_uves(self, prouters):
        with self._sem:
            for prouter in prouters:
                self.uve.delete(prouter.name)

    def sighup_handler(self):
        if self._config._args.conf_file:
            config = ConfigParser.SafeConfigParser()
            config.read(self._config._args.conf_file)
            if 'DEFAULTS' in config.sections():
                try:
                    collectors = config.get('DEFAULTS', 'collectors')
                    if type(collectors) is str:
                        collectors = collectors.split()
                        new_chksum = hashlib.md5(
                            "".join(collectors)).hexdigest()
                        if new_chksum != self._chksum:
                            self._chksum = new_chksum
                            self._config.random_collectors = \
                                random.sample(collectors, len(collectors))
                        # Reconnect to achieve load-balance irrespective of list
                        self.uve.sandesh_reconfig_collectors(
                            self._config.random_collectors)
                except ConfigParser.NoOptionError as e:
                    pass
            if 'API_SERVER' in config.sections():
                try:
                    api_servers = config.get('API_SERVER', 'api_server_list')
                except ConfigParser.NoOptionError:
                    pass
                else:
                    if isinstance(api_servers, basestring):
                        api_servers = api_servers.split()
                    new_api_server_checksum = hashlib.md5(
                        "".join(api_servers)).hexdigest()
                    if new_api_server_checksum != self._api_server_checksum:
                        self._api_server_checksum = new_api_server_checksum
                        random_api_servers = random.sample(
                            api_servers, len(api_servers))
                        self._config.set_api_server_list(random_api_servers)
                        self._vnc = None

    # end sighup_handler

    def run(self):
        """ @sighup
        SIGHUP handler to indicate configuration changes 
        """
        gevent.signal(signal.SIGHUP, self.sighup_handler)

        self._sem = Semaphore()
        self.constnt_schdlr = ConsistentScheduler(
            self.uve._moduleid,
            zookeeper=self._config.zookeeper_server(),
            delete_hndlr=self._del_uves,
            logger=self._logger,
            cluster_id=self._config.cluster_id())

        while self._keep_running:
            if not self._vnc:
                self._vnc = self._config.vnc_api()
            self.scan_data()
            if self.constnt_schdlr.schedule(self.prouters):
                members = self.constnt_schdlr.members()
                partitions = self.constnt_schdlr.partitions()
                prouters = map(lambda x: x.name,
                               self.constnt_schdlr.work_items())
                self._send_topology_uve(members, partitions, prouters)
                try:
                    with self._sem:
                        self.compute()
                        self.send_uve()
                except Exception as e:
                    traceback.print_exc()
                    print str(e)
                gevent.sleep(self._sleep_time)
            else:
                gevent.sleep(1)
        self.constnt_schdlr.finish()
Exemplo n.º 14
0
class Controller(object):
    def __init__(self, config):
        self._config = config
        self._config.random_collectors = self._config.collectors()
        self._chksum = ""
        if self._config.collectors():
             self._chksum = hashlib.md5("".join(self._config.collectors())).hexdigest()
             self._config.random_collectors = random.sample(self._config.collectors(), \
                                                            len(self._config.collectors()))
        self.uve = SnmpUve(self._config)
        self._sandesh = self.uve.sandesh_instance()
        self._hostname = socket.gethostname()
        self._logger = self.uve.logger()
        self.sleep_time()
        self.last = set()
        self._sem = Semaphore()
        self._config.set_cb(self.notify)
        self._mnt = MaxNinTtime(3, self._sleep_time)
        self._state = 'full_scan' # replace it w/ fsm
        self._if_data = None # replace it w/ fsm
        self._cleanup = None
        self._members = None
        self._partitions = None
        self._prouters = {}
        self._config_handler = SnmpConfigHandler(self._sandesh,
            self._config.rabbitmq_params(), self._config.cassandra_params())
        self._consistent_scheduler = ConsistentScheduler(self._config._name,
            zookeeper=self._config.zookeeper_server(),
            delete_hndlr=self._del_uves, logger=self._logger,
            cluster_id=self._config.cluster_id())

    def _make_if_cdata(self, data):
        if_cdata = {}
        t = time.time()
        for dev in data:
            if 'snmp' in data[dev]:
                if 'ifMib' in data[dev]['snmp']:
                    if 'ifTable' in data[dev]['snmp']['ifMib']:
                        if_cdata[dev] = dict(map(lambda x: (
                                x['ifIndex'], (x['ifOperStatus'], t)),
                                        filter(lambda x: 'ifOperStatus' in x\
                                            and 'ifDescr' in x, data[dev][
                                               'snmp']['ifMib']['ifTable'])))
                elif 'ifOperStatus' in data[dev]['snmp']:
                    if_cdata[dev] = dict((k, (v, t)) for k, v in
                                    data[dev]['snmp']['ifOperStatus'].items())
        return if_cdata

    def _delete_if_data(self, dev):
        if dev in self._if_data:
            del self._if_data[dev]

    def _set_status(self, _dict, dev, intf, val):
        if dev not in _dict:
            _dict[dev] = {}
        _dict[dev][intf] = val

    def _check_and_update_ttl(self, up2down):
        t = time.time()
        expry = 3 * self._fast_scan_freq
        for dev in self._if_data:
            for intf in self._if_data[dev]:
                if self._if_data[dev][intf][0] == 1:
                    if t - self._if_data[dev][intf][1] > expry:
                        self._set_status(up2down, dev, intf, 7) #no resp
                        self._if_data[dev][intf] = (7, t)

    def _get_if_changes(self, if_cdata):
        down2up, up2down, others = {}, {}, {}
        for dev in if_cdata:
            if dev in self._if_data:
                for intf in if_cdata[dev]:
                    if intf in self._if_data[dev]:
                        if if_cdata[dev][intf][0] != self._if_data[dev][
                                intf][0]:
                            if self._if_data[dev][intf][0] == 1:
                                self._set_status(up2down, dev, intf,
                                                 if_cdata[dev][intf][0])
                            elif if_cdata[dev][intf][0] == 1:
                                self._set_status(down2up, dev, intf,
                                                 if_cdata[dev][intf][0])
                            else:
                                self._set_status(others, dev, intf,
                                                 if_cdata[dev][intf][0])
                    self._if_data[dev][intf] = if_cdata[dev][intf]
            else:
                self._if_data[dev] = if_cdata[dev]
                for intf in self._if_data[dev]:
                    if self._if_data[dev][intf][0] == 1:
                        self._set_status(down2up, dev, intf,
                                         if_cdata[dev][intf][0])
                    else:
                        self._set_status(others, dev, intf,
                                         if_cdata[dev][intf][0])
        return down2up, up2down, others

    def _chk_if_change(self, data):
        if_cdata = self._make_if_cdata(data)
        down2up, up2down, others = self._get_if_changes(if_cdata)
        self._check_and_update_ttl(up2down)
        self._logger.debug('@chk_if_change: down2up(%s), up2down(%s), ' \
                'others(%s)' % (', '.join(down2up.keys()),
                                ', '.join(up2down.keys()),
                                ', '.join(others.keys())))
        return down2up, up2down, others

    def _extra_call_params(self):
        if self._state != 'full_scan':
            return dict(restrict='ifOperStatus')
        return {}

    def _analyze(self, data):
        ret = True
        time = self._fast_scan_freq
        if self._state != 'full_scan':
            down2up, up2down, others = self._chk_if_change(data)
            if down2up:
                self._state = 'full_scan'
                time = self._mnt.add()
            elif self._mnt.ready4full_scan():
                self._state = 'full_scan'
                time = 0
            elif up2down:
                self.uve.send_ifstatus_update(self._if_data)
            ret = False
            sret = 'chngd: ' + self._state + ', time: ' + str(time)
        else:
            self._state = 'fast_scan'
            self._if_data = self._make_if_cdata(data)
            self.uve.send_ifstatus_update(self._if_data)
            sret = 'chngd: %d' % len(self._if_data)
        self._logger.debug('@do_work(analyze):State %s(%d)->%s!' % (
                    self._state, len(data), str(sret)))
        return ret, time

    def notify(self, svc, msg='', up=True, servers=''):
        self.uve.conn_state_notify(svc, msg, up, servers)

    def sleep_time(self, newtime=None):
        if newtime:
            self._sleep_time = newtime
        else:
            self._sleep_time = self._config.frequency()
        self._fast_scan_freq = self._config.fast_scan_freq()
        if self._fast_scan_freq > self._sleep_time / 2:
            self._fast_scan_freq = self._sleep_time / 2
        return self._sleep_time

    def _setup_io(self):
        cdir = mkdtemp()
        input_file = os.path.join(cdir, 'in.data')
        output_file = os.path.join(cdir, 'out.data')
        return cdir, input_file, output_file

    def _create_input(self, input_file, output_file, devices, i, restrict=None):
        if isinstance(devices[0], DeviceDict):
            devices = DeviceConfig.populate_cfg(devices)
        with open(input_file, 'wb') as f:
            data = dict(out=output_file,
                        netdev=devices,
                        instance=i)
            if restrict:
                data['restrict'] = restrict
            pickle.dump(data, f)
            f.flush()

    def _run_scanner(self, input_file, output_file, i):
        proc = subprocess.Popen('contrail-snmp-scanner --input %s' % (
                    input_file), shell=True,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                close_fds=True)
        self._cleanup = (proc, output_file)
        o,e = proc.communicate()
        self._cleanup = None
        self._logger.debug('@run_scanner(%d): scan done with %d\nstdout:' \
                '\n%s\nstderr:\n%s\n' % (i, proc.returncode, o, e))
        with open(output_file, 'rb') as f:
            d = pickle.load(f)
        self._logger.debug('@run_scanner(%d): loaded %s' % (i, output_file))
        return d

    def _cleanup_io(self, cdir, input_file, output_file):
        os.unlink(input_file)
        os.unlink(output_file)
        os.rmdir(cdir)

    def _send_uve(self, d):
        for dev, data in d.items():
            if dev:
                self.uve.send(data['snmp'])
                self.uve.send_flow_uve({'name': dev,
                    'flow_export_source_ip': data['flow_export_source_ip']})
                self.find_fix_name(data['name'], dev)
        self._logger.debug('@send_uve:Processed %d!' % (len(d)))

    def _send_snmp_collector_uve(self, members, partitions, prouters):
        snmp_collector_info = SnmpCollectorInfo()
        if self._members != members:
            self._members = members
            snmp_collector_info.members = members
        if self._partitions != partitions:
            self._partitions = partitions
            snmp_collector_info.partitions = partitions
        new_prouters = {p.name: p for p in prouters}
        if self._prouters.keys() != new_prouters.keys():
            deleted_prouters = [v for p, v in self._prouters.iteritems() \
                if p not in new_prouters]
            self._del_uves(deleted_prouters)
            self._prouters = new_prouters
            snmp_collector_info.prouters = self._prouters.keys()
        if snmp_collector_info != SnmpCollectorInfo():
            snmp_collector_info.name = self._hostname
            SnmpCollectorUVE(data=snmp_collector_info).send()
    # end _send_snmp_collector_uve

    def _del_uves(self, l):
        with self._sem:
            for dev in l:
                self._delete_if_data(dev.name)
                self.uve.delete(dev)

    def do_work(self, i, devices):
        self._logger.debug('@do_work(%d):started (%d)...' % (i, len(devices)))
        sleep_time = self._fast_scan_freq
        if devices:
            with self._sem:
                self._work_set = devices
                cdir, input_file, output_file = self._setup_io()
                self._create_input(input_file, output_file, devices,
                                   i, **self._extra_call_params())
                data = self._run_scanner(input_file, output_file, i)
                self._cleanup_io(cdir, input_file, output_file)
                do_send, sleep_time = self._analyze(data)
                if do_send:
                    self._send_uve(data)
                    gevent.sleep(0)
                del self._work_set
        self._logger.debug('@do_work(%d):Processed %d!' % (i, len(devices)))
        return sleep_time

    def find_fix_name(self, cfg_name, snmp_name):
        if snmp_name != cfg_name:
            self._logger.debug('@find_fix_name: snmp name %s differs from ' \
                    'configured name %s, fixed for this run' % (
                            snmp_name, cfg_name))
            for d in self._work_set:
                if d.name == cfg_name:
                    d.name = snmp_name
                    return

    def sighup_handler(self):
        if self._config._args.conf_file:
            config = ConfigParser.SafeConfigParser()
            config.read(self._config._args.conf_file)
            if 'DEFAULTS' in config.sections():
                try:
                    collectors = config.get('DEFAULTS', 'collectors')
                    if type(collectors) is str:
                        collectors = collectors.split()
                        new_chksum = hashlib.md5("".join(collectors)).hexdigest()
                        if new_chksum != self._chksum:
                            self._chksum = new_chksum
                            self._config.random_collectors = \
                                random.sample(collectors, len(collectors))
                        # Reconnect to achieve load-balance irrespective of list
                        self.uve.sandesh_reconfig_collectors(
                                self._config.random_collectors)
                except ConfigParser.NoOptionError as e: 
                    pass
    # end sighup_handler  

    def _snmp_walker(self):
        i = 0
        while True:
            self._logger.debug('@run: ittr(%d)' % i)
            devices = map(lambda e: DeviceDict(e[0].split(':')[-1], e[1].obj),
                self._config_handler.get_physical_routers())
            if self._consistent_scheduler.schedule(devices):
                members = self._consistent_scheduler.members()
                partitions = self._consistent_scheduler.partitions()
                work_items = self._consistent_scheduler.work_items()
                self._send_snmp_collector_uve(members, partitions, work_items)
                sleep_time = self.do_work(i, work_items)
                self._logger.debug('done work %s' % str(self._prouters.keys()))
                i += 1
                gevent.sleep(sleep_time)
            else:
                gevent.sleep(1)
    # end _snmp_walker

    def run(self):
        """ @sighup
        SIGHUP handler to indicate configuration changes
        """
        gevent.signal(signal.SIGHUP, self.sighup_handler)

        self.gevs = [
            gevent.spawn(self._config_handler.start),
            gevent.spawn(self._snmp_walker)
        ]

        try:
            gevent.joinall(self.gevs)
        except KeyboardInterrupt:
            self._logger.error('Exiting on ^C')
        except gevent.GreenletExit:
            self._logger.error('Exiting on gevent-kill')
        finally:
            self._logger.error('stopping everything!')
            self.stop()
    # end run

    def stop(self):
        self.uve.killall()
        l = len(self.gevs)
        for i in range(0, l):
            self._logger.error('killing %d of %d' % (i+1, l))
            self.gevs[0].kill()
            self._logger.error('joining %d of %d' % (i+1, l))
            self.gevs[0].join()
            self._logger.error('stopped %d of %d' % (i+1, l))
            self.gevs.pop(0)
        self._consistent_scheduler.finish()
Exemplo n.º 15
0
class Controller(object):
    def __init__(self, config):
        self._config = config
        self._me = socket.gethostname() + ':' + str(os.getpid())
        self.analytic_api = AnalyticApiClient(self._config)
        self.uve = LinkUve(self._config)
        self.sleep_time()
        self._keep_running = True

    def stop(self):
        self._keep_running = False

    def sleep_time(self, newtime=None):
        if newtime:
            self._sleep_time = newtime
        else:
            self._sleep_time = self._config.frequency()
        return self._sleep_time

    def get_vrouters(self):
        self.analytic_api.get_vrouters(True)
        self.vrouters = {}
        self.vrouter_ips = {}
        self.vrouter_macs = {}
        for vr in self.analytic_api.list_vrouters():
            d = self.analytic_api.get_vrouter(vr, 'VrouterAgent:phy_if')
            if 'VrouterAgent' not in d:
                d['VrouterAgent'] = {}
            _ipl = self.analytic_api.get_vrouter(vr,
                                                 'VrouterAgent:self_ip_list')
            if 'VrouterAgent' in _ipl:
                d['VrouterAgent'].update(_ipl['VrouterAgent'])
            if 'VrouterAgent' not in d or\
                'self_ip_list' not in d['VrouterAgent'] or\
                'phy_if' not in d['VrouterAgent']:
                continue
            self.vrouters[vr] = {
                'ips': d['VrouterAgent']['self_ip_list'],
                'if': d['VrouterAgent']['phy_if'],
            }
            for ip in d['VrouterAgent']['self_ip_list']:
                self.vrouter_ips[ip] = vr  # index
            for intf in d['VrouterAgent']['phy_if']:
                try:
                    self.vrouter_macs[intf['mac_address']] = {}
                    self.vrouter_macs[intf['mac_address']]['vrname'] = vr
                    self.vrouter_macs[
                        intf['mac_address']]['ifname'] = intf['name']
                except:
                    continue

    def get_prouters(self):
        self.analytic_api.get_prouters(True)
        self.prouters = []
        for pr in self.analytic_api.list_prouters():
            self.prouters.append(
                PRouter(pr, self.analytic_api.get_prouter(pr, 'PRouterEntry')))

    def _is_linkup(self, prouter, ifindex):
        if 'PRouterEntry' in prouter.data and \
            'ifIndexOperStatusTable' in prouter.data['PRouterEntry']:
            status = filter(
                lambda x: x['ifIndex'] == ifindex,
                prouter.data['PRouterEntry']['ifIndexOperStatusTable'])
            if status and status[0]['ifOperStatus'] == 1:
                return True
        return False

    def _add_link(self, prouter, remote_system_name, local_interface_name,
                  remote_interface_name, local_interface_index,
                  remote_interface_index, link_type):
        d = dict(remote_system_name=remote_system_name,
                 local_interface_name=local_interface_name,
                 remote_interface_name=remote_interface_name,
                 local_interface_index=local_interface_index,
                 remote_interface_index=remote_interface_index,
                 type=link_type)
        if self._is_linkup(prouter, local_interface_index):
            if prouter.name in self.link:
                self.link[prouter.name].append(d)
            else:
                self.link[prouter.name] = [d]
            return True
        return False

    def _chk_lnk(self, pre, index):
        if 'ifIndexOperStatusTable' in pre:
            for d in pre['ifIndexOperStatusTable']:
                if d['ifIndex'] == index:
                    return d['ifOperStatus'] == 1
        return False

    def compute(self):
        self.link = {}
        for prouter in self.constnt_schdlr.work_items():
            pr, d = prouter.name, prouter.data
            if 'PRouterEntry' not in d or 'ifTable' not in d['PRouterEntry']:
                continue
            self.link[pr] = []
            lldp_ints = []
            ifm = dict(
                map(lambda x: (x['ifIndex'], x['ifDescr']),
                    d['PRouterEntry']['ifTable']))
            for pl in d['PRouterEntry']['lldpTable']['lldpRemoteSystemsData']:
                if pl['lldpRemLocalPortNum'] in ifm and self._chk_lnk(
                        d['PRouterEntry'], pl['lldpRemLocalPortNum']):
                    if pl['lldpRemPortId'].isdigit():
                        rii = int(pl['lldpRemPortId'])
                    else:
                        try:
                            rii = filter(lambda y: y['ifName'] == pl[
                                    'lldpRemPortId'], [ x for x in self.prouters \
                                    if x.name == pl['lldpRemSysName']][0].data[
                                    'PRouterEntry']['ifXTable'])[0]['ifIndex']
                        except:
                            rii = 0

                    if self._add_link(
                            prouter=prouter,
                            remote_system_name=pl['lldpRemSysName'],
                            local_interface_name=ifm[
                                pl['lldpRemLocalPortNum']],
                            remote_interface_name=pl['lldpRemPortDesc'],
                            local_interface_index=pl['lldpRemLocalPortNum'],
                            remote_interface_index=rii,
                            link_type=1):
                        lldp_ints.append(ifm[pl['lldpRemLocalPortNum']])

            vrouter_neighbors = []
            if 'fdbPortIfIndexTable' in d['PRouterEntry']:
                dot1d2snmp = map(
                    lambda x: (x['dot1dBasePortIfIndex'], x['snmpIfIndex']),
                    d['PRouterEntry']['fdbPortIfIndexTable'])
                dot1d2snmp_dict = dict(dot1d2snmp)
                if 'fdbPortTable' in d['PRouterEntry']:
                    for mac_entry in d['PRouterEntry']['fdbPortTable']:
                        if mac_entry['mac'] in self.vrouter_macs:
                            vrouter_mac_entry = self.vrouter_macs[
                                mac_entry['mac']]
                            fdbport = mac_entry['dot1dBasePortIfIndex']
                            try:
                                snmpport = dot1d2snmp_dict[fdbport]
                                ifname = ifm[snmpport]
                            except:
                                continue
                            is_lldp_int = any(ifname == lldp_int
                                              for lldp_int in lldp_ints)
                            if is_lldp_int:
                                continue
                            if self._add_link(
                                    prouter=prouter,
                                    remote_system_name=vrouter_mac_entry[
                                        'vrname'],
                                    local_interface_name=ifname,
                                    remote_interface_name=vrouter_mac_entry[
                                        'ifname'],
                                    local_interface_index=snmpport,
                                    remote_interface_index=
                                    1,  #dont know TODO:FIX
                                    link_type=2):
                                vrouter_neighbors.append(
                                    vrouter_mac_entry['vrname'])
            for arp in d['PRouterEntry']['arpTable']:
                if arp['ip'] in self.vrouter_ips:
                    if arp['mac'] in map(
                            lambda x: x['mac_address'],
                            self.vrouters[self.vrouter_ips[arp['ip']]]['if']):
                        vr_name = arp['ip']
                        vr = self.vrouters[self.vrouter_ips[vr_name]]
                        if self.vrouter_ips[vr_name] in vrouter_neighbors:
                            continue
                        if ifm[arp['localIfIndex']].startswith('vlan'):
                            continue
                        if ifm[arp['localIfIndex']].startswith('irb'):
                            continue
                        is_lldp_int = any(ifm[arp['localIfIndex']] == lldp_int
                                          for lldp_int in lldp_ints)
                        if is_lldp_int:
                            continue
                        if self._add_link(
                                prouter=prouter,
                                remote_system_name=self.vrouter_ips[vr_name],
                                local_interface_name=ifm[arp['localIfIndex']],
                                remote_interface_name=vr['if'][-1]
                            ['name'],  #TODO
                                local_interface_index=arp['localIfIndex'],
                                remote_interface_index=1,  #dont know TODO:FIX
                                link_type=2):
                            pass

    def send_uve(self):
        self.uve.send(self.link)

    def switcher(self):
        gevent.sleep(0)

    def scan_data(self):
        t = []
        t.append(gevent.spawn(self.get_vrouters))
        t.append(gevent.spawn(self.get_prouters))
        gevent.joinall(t)

    def _del_uves(self, prouters):
        with self._sem:
            for prouter in prouters:
                self.uve.delete(prouter.name)

    def run(self):
        self._sem = Semaphore()
        self.constnt_schdlr = ConsistentScheduler(
            self.uve._moduleid,
            zookeeper=self._config.zookeeper_server(),
            delete_hndlr=self._del_uves)
        while self._keep_running:
            self.scan_data()
            if self.constnt_schdlr.schedule(self.prouters):
                try:
                    with self._sem:
                        self.compute()
                        self.send_uve()
                except Exception as e:
                    import traceback
                    traceback.print_exc()
                    print str(e)
                gevent.sleep(self._sleep_time)
            else:
                gevent.sleep(1)
        self.constnt_schdlr.finish()
Exemplo n.º 16
0
class Controller(object):
    def __init__(self, config):
        self._config = config
        self._me = socket.gethostname() + ':' + str(os.getpid())
        self.analytic_api = AnalyticApiClient(self._config)
        self.uve = LinkUve(self._config)
        self.sleep_time()
        self._keep_running = True

    def stop(self):
        self._keep_running = False

    def sleep_time(self, newtime=None):
        if newtime:
            self._sleep_time = newtime
        else:
            self._sleep_time = self._config.frequency()
        return self._sleep_time

    def get_vrouters(self):
        self.analytic_api.get_vrouters(True)
        self.vrouters = {}
        self.vrouter_ips = {}
        self.vrouter_macs = {}
        for vr in self.analytic_api.list_vrouters():
            d = self.analytic_api.get_vrouter(vr, 'VrouterAgent:phy_if')
            d.update(self.analytic_api.get_vrouter(vr,
                        'VrouterAgent:self_ip_list'))
            if 'VrouterAgent' not in d or\
                'self_ip_list' not in d['VrouterAgent'] or\
                'phy_if' not in d['VrouterAgent']:
                continue
            self.vrouters[vr] = {'ips': d['VrouterAgent']['self_ip_list'],
                'if': d['VrouterAgent']['phy_if'],
            }
            for ip in d['VrouterAgent']['self_ip_list']:
                self.vrouter_ips[ip] = vr # index
            for intf in d['VrouterAgent']['phy_if']:
                try:
                    self.vrouter_macs[intf['mac_address']] = {}
                    self.vrouter_macs[intf['mac_address']]['vrname'] = vr
                    self.vrouter_macs[intf['mac_address']]['ifname'] = intf['name']
                except:
                    continue

    def get_prouters(self):
        self.analytic_api.get_prouters(True)
        self.prouters = []
        for pr in self.analytic_api.list_prouters():
            self.prouters.append(PRouter(pr, self.analytic_api.get_prouter(
                            pr, 'PRouterEntry')))

    def _is_linkup(self, prouter, ifindex):
        if 'PRouterEntry' in prouter.data and \
            'ifIndexOperStatusTable' in prouter.data['PRouterEntry']:
                status = filter(lambda x: x['ifIndex'] == ifindex,
                        prouter.data['PRouterEntry']['ifIndexOperStatusTable'])
                if status and status[0]['ifOperStatus'] == 1:
                    return True
        return False

    def _add_link(self, prouter, remote_system_name, local_interface_name,
                  remote_interface_name, local_interface_index,
                  remote_interface_index, link_type):
        d = dict(remote_system_name=remote_system_name,
                 local_interface_name=local_interface_name,
                 remote_interface_name=remote_interface_name,
                 local_interface_index=local_interface_index,
                 remote_interface_index=remote_interface_index,
                 type=link_type)
        if self._is_linkup(prouter, local_interface_index):
            if prouter.name in self.link:
                self.link[prouter.name].append(d)
            else:
                self.link[prouter.name] = [d]
            return True
        return False

    def compute(self):
        self.link = {}
        for prouter in self.constnt_schdlr.work_items():
            pr, d = prouter.name, prouter.data
            if 'PRouterEntry' not in d or 'ifTable' not in d['PRouterEntry']:
                continue
            self.link[pr] = []
            lldp_ints = []
            ifm = dict(map(lambda x: (x['ifIndex'], x['ifDescr']),
                        d['PRouterEntry']['ifTable']))
            for pl in d['PRouterEntry']['lldpTable']['lldpRemoteSystemsData']:
                if pl['lldpRemLocalPortNum'] in ifm and \
                        pl['lldpRemPortId'].isdigit():
                    if self._add_link(
                            prouter=prouter,
                            remote_system_name=pl['lldpRemSysName'],
                            local_interface_name=ifm[pl['lldpRemLocalPortNum']],
                            remote_interface_name=pl['lldpRemPortDesc'],
                            local_interface_index=pl['lldpRemLocalPortNum'],
                            remote_interface_index=int(pl['lldpRemPortId']),
                            link_type=1):
                        lldp_ints.append(ifm[pl['lldpRemLocalPortNum']])

            vrouter_neighbors = []
            if 'fdbPortIfIndexTable' in d['PRouterEntry']:
                dot1d2snmp = map (lambda x: (
                            x['dot1dBasePortIfIndex'],
                            x['snmpIfIndex']),
                        d['PRouterEntry']['fdbPortIfIndexTable'])
                dot1d2snmp_dict = dict(dot1d2snmp)
                if 'fdbPortTable' in d['PRouterEntry']:
                    for mac_entry in d['PRouterEntry']['fdbPortTable']:
                        if mac_entry['mac'] in self.vrouter_macs:
                            vrouter_mac_entry = self.vrouter_macs[mac_entry['mac']]
                            fdbport = mac_entry['dot1dBasePortIfIndex']
                            try:
                                snmpport = dot1d2snmp_dict[fdbport]
                                ifname = ifm[snmpport]
                            except:
                                continue
                            is_lldp_int = any(ifname == lldp_int for lldp_int in lldp_ints)
                            if is_lldp_int:
                                continue
                            if self._add_link(
                                    prouter=prouter,
                                    remote_system_name=vrouter_mac_entry['vrname'],
                                    local_interface_name=ifname,
                                    remote_interface_name=vrouter_mac_entry[
                                                'ifname'],
                                    local_interface_index=snmpport,
                                    remote_interface_index=1, #dont know TODO:FIX
                                    link_type=2):
                                vrouter_neighbors.append(
                                        vrouter_mac_entry['vrname'])
            for arp in d['PRouterEntry']['arpTable']:
                if arp['ip'] in self.vrouter_ips:
                    if arp['mac'] in map(lambda x: x['mac_address'],
                            self.vrouters[self.vrouter_ips[arp['ip']]]['if']):
                        vr_name = arp['ip']
                        vr = self.vrouters[self.vrouter_ips[vr_name]]
                        if self.vrouter_ips[vr_name] in vrouter_neighbors:
                            continue
                        if ifm[arp['localIfIndex']].startswith('vlan'):
                            continue
                        if ifm[arp['localIfIndex']].startswith('irb'):
                            continue
                        is_lldp_int = any(ifm[arp['localIfIndex']] == lldp_int for lldp_int in lldp_ints)
                        if is_lldp_int:
                            continue
                        if self._add_link(
                                prouter=prouter,
                                remote_system_name=self.vrouter_ips[vr_name],
                                local_interface_name=ifm[arp['localIfIndex']],
                                remote_interface_name=vr['if'][-1]['name'],#TODO
                                local_interface_index=arp['localIfIndex'],
                                remote_interface_index=1, #dont know TODO:FIX
                                link_type=2):
                            pass

    def send_uve(self):
        self.uve.send(self.link)

    def switcher(self):
        gevent.sleep(0)

    def scan_data(self):
        t = []
        t.append(gevent.spawn(self.get_vrouters))
        t.append(gevent.spawn(self.get_prouters))
        gevent.joinall(t)

    def _del_uves(self, prouters):
        with self._sem:
            for prouter in prouters:
                self.uve.delete(prouter.name)

    def run(self):
        self._sem = Semaphore()
        self.constnt_schdlr = ConsistentScheduler(
                            self.uve._moduleid,
                            zookeeper=self._config.zookeeper_server(),
                            delete_hndlr=self._del_uves)
        while self._keep_running:
            self.scan_data()
            if self.constnt_schdlr.schedule(self.prouters):
                try:
                    with self._sem:
                        self.compute()
                        self.send_uve()
                except Exception as e:
                    import traceback; traceback.print_exc()
                    print str(e)
                gevent.sleep(self._sleep_time)
            else:
                gevent.sleep(1)
        self.constnt_schdlr.finish()
Exemplo n.º 17
0
class Controller(object):
    def __init__(self, config):
        self._config = config
        self._me = socket.gethostname() + ':' + str(os.getpid())
        self.analytic_api = AnalyticApiClient(self._config)
        self.uve = LinkUve(self._config)
        self.sleep_time()
        self._keep_running = True

    def stop(self):
        self._keep_running = False

    def sleep_time(self, newtime=None):
        if newtime:
            self._sleep_time = newtime
        else:
            self._sleep_time = self._config.frequency()
        return self._sleep_time

    def get_vrouters(self):
        self.analytic_api.get_vrouters(True)
        self.vrouters = {}
        self.vrouter_ips = {}
        self.vrouter_macs = {}
        for vr in self.analytic_api.list_vrouters():
            d = self.analytic_api.get_vrouter(vr)
            if 'VrouterAgent' not in d or\
                'self_ip_list' not in d['VrouterAgent'] or\
                'phy_if' not in d['VrouterAgent']:
                continue
            self.vrouters[vr] = {'ips': d['VrouterAgent']['self_ip_list'],
                'if': d['VrouterAgent']['phy_if'],
            }
            for ip in d['VrouterAgent']['self_ip_list']:
                self.vrouter_ips[ip] = vr # index
            for intf in d['VrouterAgent']['phy_if']:
                try:
                    self.vrouter_macs[intf['mac_address']] = {}
                    self.vrouter_macs[intf['mac_address']]['vrname'] = vr
                    self.vrouter_macs[intf['mac_address']]['ifname'] = intf['name']
                except:
                    continue

    def get_prouters(self):
        self.analytic_api.get_prouters(True)
        self.prouters = []
        for vr in self.analytic_api.list_prouters():
            self.prouters.append(PRouter(vr, self.analytic_api.get_prouter(vr)))

    def compute(self):
        self.link = {}
        for prouter in self.constnt_schdlr.work_items():
            pr, d = prouter.name, prouter.data
            if 'PRouterEntry' not in d or 'ifTable' not in d['PRouterEntry']:
                continue
            self.link[pr] = []
            lldp_ints = []
            ifm = dict(map(lambda x: (x['ifIndex'], x['ifDescr']),
                        d['PRouterEntry']['ifTable']))
            for pl in d['PRouterEntry']['lldpTable']['lldpRemoteSystemsData']:
                if pl['lldpRemLocalPortNum'] in ifm and\
                    pl['lldpRemPortId'].isdigit():
                    self.link[pr].append({
                        'remote_system_name': pl['lldpRemSysName'],
                        'local_interface_name': ifm[pl['lldpRemLocalPortNum']],
                        'remote_interface_name': pl['lldpRemPortDesc'],
                        'local_interface_index': pl['lldpRemLocalPortNum'],
                        'remote_interface_index': int(pl['lldpRemPortId']),
                        'type': 1
                        })
                    lldp_ints.append(ifm[pl['lldpRemLocalPortNum']])

            vrouter_neighbors = []
            if 'fdbPortIfIndexTable' in d['PRouterEntry']:
                dot1d2snmp = map (lambda x: (x['dot1dBasePortIfIndex'], x['snmpIfIndex']), d['PRouterEntry']['fdbPortIfIndexTable'])
                dot1d2snmp_dict = dict(dot1d2snmp)
                if 'fdbPortTable' in d['PRouterEntry']:
                    for mac_entry in d['PRouterEntry']['fdbPortTable']:
                        if mac_entry['mac'] in self.vrouter_macs:
                            vrouter_mac_entry = self.vrouter_macs[mac_entry['mac']]
                            fdbport = mac_entry['dot1dBasePortIfIndex']
                            try:
                                snmpport = dot1d2snmp_dict[fdbport]
                            except:
                                continue
                            is_lldp_int = any(ifm[snmpport] == lldp_int for lldp_int in lldp_ints)
                            if is_lldp_int:
                                continue
                            self.link[pr].append({
                                'remote_system_name': vrouter_mac_entry['vrname'],
                                'local_interface_name': ifm[snmpport],
                                'remote_interface_name': vrouter_mac_entry['ifname'],
                                'local_interface_index': snmpport,
                                'remote_interface_index': 1, #dont know TODO:FIX
                                'type': 2
                                    })
                            vrouter_neighbors.append(vrouter_mac_entry['vrname'])
            for arp in d['PRouterEntry']['arpTable']:
                if arp['ip'] in self.vrouter_ips:
                    if arp['mac'] in map(lambda x: x['mac_address'],
                            self.vrouters[self.vrouter_ips[arp['ip']]]['if']):
                        vr_name = arp['ip']
                        vr = self.vrouters[self.vrouter_ips[vr_name]]
                        if self.vrouter_ips[vr_name] in vrouter_neighbors:
                            continue
                        if ifm[arp['localIfIndex']].startswith('vlan'):
                            continue
                        if ifm[arp['localIfIndex']].startswith('irb'):
                            continue
                        is_lldp_int = any(ifm[arp['localIfIndex']] == lldp_int for lldp_int in lldp_ints)
                        if is_lldp_int:
                            continue
                        self.link[pr].append({
                            'remote_system_name': self.vrouter_ips[vr_name],
                            'local_interface_name': ifm[arp['localIfIndex']],
                            'remote_interface_name': vr['if'][-1]['name'],#TODO
                            'local_interface_index': arp['localIfIndex'],
                            'remote_interface_index': 1, #dont know TODO:FIX
                            'type': 2
                                })

    def send_uve(self):
        self.uve.send(self.link)

    def switcher(self):
        gevent.sleep(0)

    def scan_data(self):
        t = []
        t.append(gevent.spawn(self.get_vrouters))
        t.append(gevent.spawn(self.get_prouters))
        gevent.joinall(t)

    def _del_uves(self, prouters):
        with self._sem:
            for prouter in prouters:
                self.uve.delete(prouter.name)

    def run(self):
        self._sem = Semaphore()
        self.constnt_schdlr = ConsistentScheduler(
                            self.uve._moduleid,
                            zookeeper=self._config.zookeeper_server(),
                            delete_hndlr=self._del_uves)
        while self._keep_running:
            self.scan_data()
            if self.constnt_schdlr.schedule(self.prouters):
                try:
                    with self._sem:
                        self.compute()
                        self.send_uve()
                except Exception as e:
                    import traceback; traceback.print_exc()
                    print str(e)
                gevent.sleep(self._sleep_time)
            else:
                gevent.sleep(1)
        self.constnt_schdlr.finish()
Exemplo n.º 18
0
class Controller(object):
    def __init__(self, config):
        self._config = config
        if 'host_ip' in self._config._args:
            host_ip = self._config._args.host_ip
        else:
            host_ip = socket.gethostbyname(socket.getfqdn())
        self._hostname = socket.getfqdn(host_ip)
        self.analytic_api = AnalyticApiClient(self._config)
        self._config.random_collectors = self._config.collectors()
        self._chksum = ""
        if self._config.collectors():
            self._chksum = hashlib.md5("".join(
                self._config.collectors())).hexdigest()
            self._config.random_collectors = random.sample(self._config.collectors(), \
                                                           len(self._config.collectors()))
        self.uve = LinkUve(self._config)
        self._sandesh = self.uve.sandesh_instance()
        self._logger = self.uve.logger()
        self.sleep_time()
        self._sem = Semaphore()
        self._members = None
        self._partitions = None
        self._prouters = {}
        self._vrouter_l2ifs = {}
        self._old_vrouter_l2ifs = {}
        self._config_handler = TopologyConfigHandler(
            self._sandesh, self._config.rabbitmq_params(),
            self._config.cassandra_params(), host_ip)
        self.constnt_schdlr = ConsistentScheduler(
            self.uve._moduleid,
            zookeeper=self._config.zookeeper_server(),
            delete_hndlr=self._del_uves,
            logger=self._logger,
            cluster_id=self._config.cluster_id())

    def sleep_time(self, newtime=None):
        if newtime:
            self._sleep_time = newtime
        else:
            self._sleep_time = self._config.frequency()
        return self._sleep_time

    def get_vrouters(self):
        self.analytic_api.get_vrouters(True)
        self.vrouters = {}
        self.vrouter_ips = {}
        self.vrouter_macs = {}
        for vr in self.analytic_api.list_vrouters():
            cfilt = [
                'VrouterAgent:phy_if', 'VrouterAgent:self_ip_list',
                'VRouterL2IfInfo'
            ]
            try:
                d = self.analytic_api.get_vrouter(vr, ','.join(cfilt))
            except Exception as e:
                traceback.print_exc()
                print(str(e))
                d = {}
            if 'VrouterAgent' not in d or\
                'self_ip_list' not in d['VrouterAgent'] or\
                'phy_if' not in d['VrouterAgent']:
                continue
            self.vrouters[vr] = {
                'ips': d['VrouterAgent']['self_ip_list'],
                'if': d['VrouterAgent']['phy_if']
            }
            try:
                self.vrouters[vr]['l2_if'] = d['VRouterL2IfInfo']['if_info']
            except KeyError:
                pass
            for ip in d['VrouterAgent']['self_ip_list']:
                self.vrouter_ips[ip] = vr  # index
            for intf in d['VrouterAgent']['phy_if']:
                try:
                    self.vrouter_macs[intf['mac_address']] = {}
                    self.vrouter_macs[intf['mac_address']]['vrname'] = vr
                    self.vrouter_macs[
                        intf['mac_address']]['ifname'] = intf['name']
                except:
                    continue

    def get_prouters(self):
        self.analytic_api.get_prouters(True)
        self.prouters = []
        for pr in self.analytic_api.list_prouters():
            try:
                data = self.analytic_api.get_prouter(pr, 'PRouterEntry')
                if data:
                    self.prouters.append(PRouter(pr, data))
            except Exception as e:
                traceback.print_exc()
                print(str(e))

    def _is_linkup(self, prouter, ifindex):
        if 'PRouterEntry' in prouter.data and \
            'ifIndexOperStatusTable' in prouter.data['PRouterEntry']:
            status = [
                x
                for x in prouter.data['PRouterEntry']['ifIndexOperStatusTable']
                if x['ifIndex'] == ifindex
            ]
            if status and status[0]['ifOperStatus'] == 1:
                return True
        return False

    def _add_link(self, prouter, remote_system_name, local_interface_name,
                  remote_interface_name, local_interface_index,
                  remote_interface_index, link_type):
        # If the remote_system_name or remote_interface_name is None, do not
        # add this link in the link_table.
        if not all([remote_system_name, remote_interface_name]):
            return False
        d = dict(remote_system_name=remote_system_name,
                 local_interface_name=local_interface_name,
                 remote_interface_name=remote_interface_name,
                 local_interface_index=local_interface_index,
                 remote_interface_index=remote_interface_index,
                 type=link_type)
        if link_type == RemoteType.VRouter:
            l2_if = self.vrouters[remote_system_name].get('l2_if')
            if l2_if and remote_interface_name in l2_if:
                if l2_if[remote_interface_name]['remote_system_name'] != \
                        prouter.name:
                    return False
        if self._is_linkup(prouter, local_interface_index):
            if prouter.name in self.link:
                self.link[prouter.name].append(d)
            else:
                self.link[prouter.name] = [d]
            return True
        return False

    def _chk_lnk(self, pre, index):
        if 'ifIndexOperStatusTable' in pre:
            for d in pre['ifIndexOperStatusTable']:
                if d['ifIndex'] == index:
                    return d['ifOperStatus'] == 1
        return False

    def _send_topology_uve(self, members, partitions, prouters):
        topology_info = TopologyInfo()
        if self._members != members:
            self._members = members
            topology_info.members = members
        if self._partitions != partitions:
            self._partitions = partitions
            topology_info.partitions = partitions
        new_prouters = {p.name: p for p in prouters}
        if list(self._prouters.keys()) != list(new_prouters.keys()):
            deleted_prouters = [v for p, v in self._prouters.items() \
                if p not in new_prouters]
            self._del_uves(deleted_prouters)
            self._prouters = new_prouters
            topology_info.prouters = list(self._prouters.keys())
        if topology_info != TopologyInfo():
            topology_info.name = self._hostname
            TopologyUVE(data=topology_info).send()

    # end _send_topology_uve

    def bms_links(self, prouter, ifm):
        try:
            lifs = self._config_handler.get_logical_interfaces()
            if not lifs:
                return
            for lif_fqname, lif in lifs:
                if prouter.name in lif_fqname:
                    vmi_refs = lif.obj.get_virtual_machine_interface_refs()
                    if not vmi_refs:
                        continue
                    for vmif in vmi_refs:
                        vmi = self._config_handler.\
                                get_virtual_machine_interface(fq_name=None,
                                                              uuid=vmif['uuid'])
                        if not vmi:
                            continue
                        vmi = vmi.obj
                        macs = vmi.virtual_machine_interface_mac_addresses.\
                               get_mac_address()
                        if not macs:
                            continue
                        for mc in macs:
                            ifi = [k for k in ifm if ifm[k] in lif_fqname][0]
                            rsys = '-'.join(['bms', 'host'] + mc.split(':'))
                            self._add_link(
                                prouter=prouter,
                                remote_system_name=rsys,
                                local_interface_name=lif.obj.fq_name[-1],
                                remote_interface_name='em0',  #no idea
                                local_interface_index=ifi,
                                remote_interface_index=1,  #dont know TODO:FIX
                                link_type=RemoteType.BMS)
        except:
            traceback.print_exc()

    def compute(self):
        self.link = {}
        self._old_vrouter_l2ifs = self._vrouter_l2ifs
        self._vrouter_l2ifs = {}
        for prouter in self.constnt_schdlr.work_items():
            pr, d = prouter.name, prouter.data
            if 'PRouterEntry' not in d or 'ifTable' not in d['PRouterEntry']:
                continue
            self.link[pr] = []
            lldp_ints = []
            ifm = dict([(x['ifIndex'], x['ifDescr'])
                        for x in d['PRouterEntry']['ifTable']])
            self.bms_links(prouter, ifm)
            for pl in d['PRouterEntry']['lldpTable']['lldpRemoteSystemsData']:
                if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                        'lldpLocSysDesc'].startswith('Cisco'):
                    loc_pname = [
                        x for x in d['PRouterEntry']['lldpTable']
                        ['lldpLocalSystemData']['lldpLocPortTable']
                        if x['lldpLocPortNum'] == pl['lldpRemLocalPortNum']
                    ][0]['lldpLocPortDesc']
                    pl['lldpRemLocalPortNum'] = [
                        k for k in ifm if ifm[k] == loc_pname
                    ][0]
                elif d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                        'lldpLocSysDesc'].startswith('Arista'):
                    loc_pname = [
                        x for x in d['PRouterEntry']['lldpTable']
                        ['lldpLocalSystemData']['lldpLocPortTable']
                        if x['lldpLocPortNum'] == pl['lldpRemLocalPortNum']
                    ][0]['lldpLocPortId']
                    pl['lldpRemLocalPortNum'] = [
                        k for k in ifm if ifm[k] == loc_pname
                    ][0]
                if pl['lldpRemLocalPortNum'] in ifm and self._chk_lnk(
                        d['PRouterEntry'], pl['lldpRemLocalPortNum']):
                    if pl['lldpRemPortId'].isdigit():
                        rii = int(pl['lldpRemPortId'])
                    else:
                        try:
                            if d['PRouterEntry']['lldpTable'][
                                    'lldpLocalSystemData'][
                                        'lldpLocSysDesc'].startswith('Arista'):
                                rpn = filter(
                                    lambda y: y['lldpLocPortId'] == pl[
                                        'lldpRemPortId'], [
                                            x for x in self.prouters
                                            if x.name == pl['lldpRemSysName']
                                        ][0].data['PRouterEntry']['lldpTable']
                                    ['lldpLocalSystemData']
                                    ['lldpLocPortTable'])[0]['lldpLocPortId']
                            else:
                                rpn = filter(
                                    lambda y: y['lldpLocPortId'] == pl[
                                        'lldpRemPortId'], [
                                            x for x in self.prouters
                                            if x.name == pl['lldpRemSysName']
                                        ][0].data['PRouterEntry']['lldpTable']
                                    ['lldpLocalSystemData']
                                    ['lldpLocPortTable'])[0]['lldpLocPortDesc']
                            rii = filter(lambda y: y['ifDescr'] == rpn,
                                    [ x for x in self.prouters \
                                    if x.name == pl['lldpRemSysName']][0].data[
                                    'PRouterEntry']['ifTable'])[0]['ifIndex']
                        except:
                            rii = 0

                    if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                            'lldpLocSysDesc'].startswith('Arista'):
                        if self._add_link(
                                prouter=prouter,
                                remote_system_name=pl['lldpRemSysName'],
                                local_interface_name=ifm[
                                    pl['lldpRemLocalPortNum']],
                                remote_interface_name=pl['lldpRemPortId'],
                                local_interface_index=pl[
                                    'lldpRemLocalPortNum'],
                                remote_interface_index=rii,
                                link_type=RemoteType.PRouter):
                            lldp_ints.append(ifm[pl['lldpRemLocalPortNum']])
                    else:
                        if self._add_link(
                                prouter=prouter,
                                remote_system_name=pl['lldpRemSysName'],
                                local_interface_name=ifm[
                                    pl['lldpRemLocalPortNum']],
                                remote_interface_name=pl['lldpRemPortDesc'],
                                local_interface_index=pl[
                                    'lldpRemLocalPortNum'],
                                remote_interface_index=rii,
                                link_type=RemoteType.PRouter):
                            lldp_ints.append(ifm[pl['lldpRemLocalPortNum']])

            vrouter_l2ifs = {}
            if 'fdbPortIfIndexTable' in d['PRouterEntry']:
                dot1d2snmp = [(x['dot1dBasePortIfIndex'], x['snmpIfIndex'])
                              for x in d['PRouterEntry']['fdbPortIfIndexTable']
                              ]
                dot1d2snmp_dict = dict(dot1d2snmp)
                if 'fdbPortTable' in d['PRouterEntry']:
                    for mac_entry in d['PRouterEntry']['fdbPortTable']:
                        if mac_entry['mac'] in self.vrouter_macs:
                            vrouter_mac_entry = self.vrouter_macs[
                                mac_entry['mac']]
                            vr_name = vrouter_mac_entry['vrname']
                            vr_ifname = vrouter_mac_entry['ifname']
                            fdbport = mac_entry['dot1dBasePortIfIndex']
                            try:
                                snmpport = dot1d2snmp_dict[fdbport]
                                ifname = ifm[snmpport]
                            except:
                                continue
                            is_lldp_int = any(ifname == lldp_int
                                              for lldp_int in lldp_ints)
                            if is_lldp_int:
                                continue
                            if self._add_link(
                                    prouter=prouter,
                                    remote_system_name=vr_name,
                                    local_interface_name=ifname,
                                    remote_interface_name=vr_ifname,
                                    local_interface_index=snmpport,
                                    remote_interface_index=
                                    1,  #dont know TODO:FIX
                                    link_type=RemoteType.VRouter):
                                if vr_name not in vrouter_l2ifs:
                                    vrouter_l2ifs[vr_name] = {}
                                vrouter_l2ifs[vr_name][vr_ifname] = {
                                    'remote_system_name': prouter.name,
                                    'remote_if_name': ifname,
                                }
            for arp in d['PRouterEntry']['arpTable']:
                if arp['ip'] in self.vrouter_ips:
                    if arp['mac'] in [
                            x['mac_address'] for x in self.vrouters[
                                self.vrouter_ips[arp['ip']]]['if']
                    ]:
                        vr_name = self.vrouter_macs[arp['mac']]['vrname']
                        vr_ifname = self.vrouter_macs[arp['mac']]['ifname']
                        try:
                            if vrouter_l2ifs[vr_name][vr_ifname]\
                                ['remote_system_name'] == prouter.name:
                                del vrouter_l2ifs[vr_name][vr_ifname]
                                if not vrouter_l2ifs[vr_name]:
                                    del vrouter_l2ifs[vr_name]
                                continue
                        except KeyError:
                            pass
                        if ifm[arp['localIfIndex']].startswith('vlan'):
                            continue
                        if ifm[arp['localIfIndex']].startswith('irb'):
                            continue
                        is_lldp_int = any(ifm[arp['localIfIndex']] == lldp_int
                                          for lldp_int in lldp_ints)
                        if is_lldp_int:
                            continue
                        if self._add_link(
                                prouter=prouter,
                                remote_system_name=vr_name,
                                local_interface_name=ifm[arp['localIfIndex']],
                                remote_interface_name=vr_ifname,
                                local_interface_index=arp['localIfIndex'],
                                remote_interface_index=1,  #dont know TODO:FIX
                                link_type=RemoteType.VRouter):
                            pass
            for vr, intf in vrouter_l2ifs.items():
                if vr in self._vrouter_l2ifs:
                    self._vrouter_l2ifs[vr].update(vrouter_l2ifs[vr])
                else:
                    self._vrouter_l2ifs[vr] = intf

    def send_uve(self):
        old_vrs = set(self._old_vrouter_l2ifs.keys())
        new_vrs = set(self._vrouter_l2ifs.keys())
        del_vrs = old_vrs - new_vrs
        add_vrs = new_vrs - old_vrs
        same_vrs = old_vrs.intersection(new_vrs)
        for vr in del_vrs:
            vr_l2info = VRouterL2IfInfo(name=vr, deleted=True)
            VRouterL2IfUVE(data=vr_l2info).send()
        for vr in add_vrs:
            if_info = {}
            for vrif, remif_info in self._vrouter_l2ifs[vr].items():
                if_info[vrif] = RemoteIfInfo(remif_info['remote_system_name'],
                                             remif_info['remote_if_name'])
            vr_l2info = VRouterL2IfInfo(name=vr, if_info=if_info)
            VRouterL2IfUVE(data=vr_l2info).send()
        for vr in same_vrs:
            if self._vrouter_l2ifs[vr] != self._old_vrouter_l2ifs[vr]:
                if_info = {}
                for vrif, remif_info in self._vrouter_l2ifs[vr].items():
                    if_info[vrif] = RemoteIfInfo(
                        remif_info['remote_system_name'],
                        remif_info['remote_if_name'])
                vr_l2info = VRouterL2IfInfo(name=vr, if_info=if_info)
                VRouterL2IfUVE(data=vr_l2info).send()
        self.uve.send(self.link)

    def switcher(self):
        gevent.sleep(0)

    def scan_data(self):
        t = []
        t.append(gevent.spawn(self.get_vrouters))
        t.append(gevent.spawn(self.get_prouters))
        gevent.joinall(t)

    def _del_uves(self, prouters):
        with self._sem:
            for prouter in prouters:
                self.uve.delete(prouter.name)

    def sighup_handler(self):
        if self._config._args.conf_file:
            config = configparser.SafeConfigParser()
            config.read(self._config._args.conf_file)
            if 'DEFAULTS' in config.sections():
                try:
                    collectors = config.get('DEFAULTS', 'collectors')
                    if isinstance(collectors, (basestring, str)):
                        collectors = collectors.split()
                        new_chksum = hashlib.md5(
                            "".join(collectors)).hexdigest()
                        if new_chksum != self._chksum:
                            self._chksum = new_chksum
                            self._config.random_collectors = \
                                random.sample(collectors, len(collectors))
                        # Reconnect to achieve load-balance irrespective of list
                        self.uve.sandesh_reconfig_collectors(
                            self._config.random_collectors)
                except configparser.NoOptionError as e:
                    pass

    # end sighup_handler

    def _uve_scanner(self):
        while True:
            self.scan_data()
            if self.constnt_schdlr.schedule(self.prouters):
                members = self.constnt_schdlr.members()
                partitions = self.constnt_schdlr.partitions()
                self._send_topology_uve(members, partitions,
                                        self.constnt_schdlr.work_items())
                try:
                    with self._sem:
                        self.compute()
                        self.send_uve()
                except Exception as e:
                    traceback.print_exc()
                    print(str(e))
                gevent.sleep(self._sleep_time)
            else:
                gevent.sleep(1)

    # end _uve_scanner

    def run(self):
        """ @sighup
        SIGHUP handler to indicate configuration changes
        """
        gevent.signal(signal.SIGHUP, self.sighup_handler)

        self.gevs = [
            gevent.spawn(self._config_handler.start),
            gevent.spawn(self._uve_scanner)
        ]

        try:
            gevent.joinall(self.gevs)
        except KeyboardInterrupt:
            self._logger.error('Exiting on ^C')
        except gevent.GreenletExit:
            self._logger.error('Exiting on gevent-kill')
        finally:
            self._logger.error('stopping everything!')
            self.stop()

    # end run

    def stop(self):
        self.uve.stop()
        l = len(self.gevs)
        for i in range(0, l):
            self._logger.error('killing %d of %d' % (i + 1, l))
            self.gevs[0].kill()
            self._logger.error('joining %d of %d' % (i + 1, l))
            self.gevs[0].join()
            self._logger.error('stopped %d of %d' % (i + 1, l))
            self.gevs.pop(0)
        self.constnt_schdlr.finish()
class Controller(object):
    def __init__(self, config):
        self._config = config
        self._me = socket.gethostname() + ':' + str(os.getpid())
        self.analytic_api = AnalyticApiClient(self._config)
        self.uve = LinkUve(self._config)
        self.sleep_time()
        self._keep_running = True
        self._vnc = None

    def stop(self):
        self._keep_running = False

    def sleep_time(self, newtime=None):
        if newtime:
            self._sleep_time = newtime
        else:
            self._sleep_time = self._config.frequency()
        return self._sleep_time

    def get_vrouters(self):
        self.analytic_api.get_vrouters(True)
        self.vrouters = {}
        self.vrouter_ips = {}
        self.vrouter_macs = {}
        for vr in self.analytic_api.list_vrouters():
            try:
                d = self.analytic_api.get_vrouter(vr, 'VrouterAgent:phy_if')
            except Exception as e:
                traceback.print_exc()
                print str(e)
                d = {}
            if 'VrouterAgent' not in d:
                d['VrouterAgent'] = {}
            try:
                _ipl = self.analytic_api.get_vrouter(vr,
                        'VrouterAgent:self_ip_list')
            except Exception as e:
                traceback.print_exc()
                print str(e)
                _ipl = {}
            if 'VrouterAgent' in _ipl:
                d['VrouterAgent'].update(_ipl['VrouterAgent'])
            if 'VrouterAgent' not in d or\
                'self_ip_list' not in d['VrouterAgent'] or\
                'phy_if' not in d['VrouterAgent']:
                continue
            self.vrouters[vr] = {'ips': d['VrouterAgent']['self_ip_list'],
                'if': d['VrouterAgent']['phy_if'],
            }
            for ip in d['VrouterAgent']['self_ip_list']:
                self.vrouter_ips[ip] = vr # index
            for intf in d['VrouterAgent']['phy_if']:
                try:
                    self.vrouter_macs[intf['mac_address']] = {}
                    self.vrouter_macs[intf['mac_address']]['vrname'] = vr
                    self.vrouter_macs[intf['mac_address']]['ifname'] = intf['name']
                except:
                    continue

    def get_prouters(self):
        self.analytic_api.get_prouters(True)
        self.prouters = []
        for pr in self.analytic_api.list_prouters():
            try:
                self.prouters.append(PRouter(pr, self.analytic_api.get_prouter(
                            pr, 'PRouterEntry')))
            except Exception as e:
                traceback.print_exc()
                print str(e)

    def _is_linkup(self, prouter, ifindex):
        if 'PRouterEntry' in prouter.data and \
            'ifIndexOperStatusTable' in prouter.data['PRouterEntry']:
                status = filter(lambda x: x['ifIndex'] == ifindex,
                        prouter.data['PRouterEntry']['ifIndexOperStatusTable'])
                if status and status[0]['ifOperStatus'] == 1:
                    return True
        return False

    def _add_link(self, prouter, remote_system_name, local_interface_name,
                  remote_interface_name, local_interface_index,
                  remote_interface_index, link_type):
        d = dict(remote_system_name=remote_system_name,
                 local_interface_name=local_interface_name,
                 remote_interface_name=remote_interface_name,
                 local_interface_index=local_interface_index,
                 remote_interface_index=remote_interface_index,
                 type=link_type)
        if self._is_linkup(prouter, local_interface_index):
            if prouter.name in self.link:
                self.link[prouter.name].append(d)
            else:
                self.link[prouter.name] = [d]
            return True
        return False

    def _chk_lnk(self, pre, index):
        if 'ifIndexOperStatusTable' in pre:
            for d in pre['ifIndexOperStatusTable']:
                if d['ifIndex'] == index:
                    return d['ifOperStatus'] == 1
        return False

    def bms_links(self, prouter, ifm):
        if not self._vnc:
            try:
                self._vnc = self._config.vnc_api()
            except:
                print 'Proceeding without any api-server'
                self._vnc = None # refresh
        if self._vnc:
            try:
                for li in self._vnc.logical_interfaces_list()[
                            'logical-interfaces']:
                    if prouter.name in li['fq_name']:
                        lif = self._vnc.logical_interface_read(id=li['uuid'])
                        for vmif in lif.get_virtual_machine_interface_refs():
                            vmi = self._vnc.virtual_machine_interface_read(
                                    id=vmif['uuid'])
                            for mc in vmi.virtual_machine_interface_mac_addresses.get_mac_address():
                                ifi = [k for k in ifm if ifm[k] in li[
                                                    'fq_name']][0]
                                rsys = '-'.join(['bms', 'host'] + mc.split(
                                            ':'))
                                if self._add_link(
                                        prouter=prouter,
                                        remote_system_name=rsys,
                                        local_interface_name=li['fq_name'][
                                                                    -1],
                                        remote_interface_name='em0',#no idea
                                        local_interface_index=ifi,
                                        remote_interface_index=1, #dont know TODO:FIX
                                        link_type=2):
                                    pass
            except:
                traceback.print_exc()
                self._vnc = None # refresh


    def compute(self):
        self.link = {}
        for prouter in self.constnt_schdlr.work_items():
            pr, d = prouter.name, prouter.data
            if 'PRouterEntry' not in d or 'ifTable' not in d['PRouterEntry']:
                continue
            self.link[pr] = []
            lldp_ints = []
            ifm = dict(map(lambda x: (x['ifIndex'], x['ifDescr']),
                        d['PRouterEntry']['ifTable']))
            self.bms_links(prouter, ifm)
            for pl in d['PRouterEntry']['lldpTable']['lldpRemoteSystemsData']:
                if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                    'lldpLocSysDesc'].startswith('Cisco'):
                    loc_pname = [x for x in d['PRouterEntry']['lldpTable'][
                            'lldpLocalSystemData']['lldpLocPortTable'] if x[
                            'lldpLocPortNum'] == pl['lldpRemLocalPortNum']][
                            0]['lldpLocPortDesc']
                    pl['lldpRemLocalPortNum'] = [k for k in ifm if ifm[
                                            k] == loc_pname][0]
                if pl['lldpRemLocalPortNum'] in ifm and self._chk_lnk(
                        d['PRouterEntry'], pl['lldpRemLocalPortNum']):
                    if pl['lldpRemPortId'].isdigit():
                        rii = int(pl['lldpRemPortId'])
                    else:
                        try:
                            rpn = filter(lambda y: y['lldpLocPortId'] == pl[
                                    'lldpRemPortId'], [
                                    x for x in self.prouters if x.name == pl[
                                    'lldpRemSysName']][0].data['PRouterEntry'][
                                    'lldpTable']['lldpLocalSystemData'][
                                    'lldpLocPortTable'])[0]['lldpLocPortDesc']
                            rii = filter(lambda y: y['ifDescr'] == rpn,
                                    [ x for x in self.prouters \
                                    if x.name == pl['lldpRemSysName']][0].data[
                                    'PRouterEntry']['ifTable'])[0]['ifIndex']
                        except:
                            rii = 0

                    if self._add_link(
                            prouter=prouter,
                            remote_system_name=pl['lldpRemSysName'],
                            local_interface_name=ifm[pl['lldpRemLocalPortNum']],
                            remote_interface_name=pl['lldpRemPortDesc'],
                            local_interface_index=pl['lldpRemLocalPortNum'],
                            remote_interface_index=rii,
                            link_type=1):
                        lldp_ints.append(ifm[pl['lldpRemLocalPortNum']])

            vrouter_neighbors = []
            if 'fdbPortIfIndexTable' in d['PRouterEntry']:
                dot1d2snmp = map (lambda x: (
                            x['dot1dBasePortIfIndex'],
                            x['snmpIfIndex']),
                        d['PRouterEntry']['fdbPortIfIndexTable'])
                dot1d2snmp_dict = dict(dot1d2snmp)
                if 'fdbPortTable' in d['PRouterEntry']:
                    for mac_entry in d['PRouterEntry']['fdbPortTable']:
                        if mac_entry['mac'] in self.vrouter_macs:
                            vrouter_mac_entry = self.vrouter_macs[mac_entry['mac']]
                            fdbport = mac_entry['dot1dBasePortIfIndex']
                            try:
                                snmpport = dot1d2snmp_dict[fdbport]
                                ifname = ifm[snmpport]
                            except:
                                continue
                            is_lldp_int = any(ifname == lldp_int for lldp_int in lldp_ints)
                            if is_lldp_int:
                                continue
                            if self._add_link(
                                    prouter=prouter,
                                    remote_system_name=vrouter_mac_entry['vrname'],
                                    local_interface_name=ifname,
                                    remote_interface_name=vrouter_mac_entry[
                                                'ifname'],
                                    local_interface_index=snmpport,
                                    remote_interface_index=1, #dont know TODO:FIX
                                    link_type=2):
                                vrouter_neighbors.append(
                                        vrouter_mac_entry['vrname'])
            for arp in d['PRouterEntry']['arpTable']:
                if arp['ip'] in self.vrouter_ips:
                    if arp['mac'] in map(lambda x: x['mac_address'],
                            self.vrouters[self.vrouter_ips[arp['ip']]]['if']):
                        vr_name = arp['ip']
                        vr = self.vrouters[self.vrouter_ips[vr_name]]
                        if self.vrouter_ips[vr_name] in vrouter_neighbors:
                            continue
                        if ifm[arp['localIfIndex']].startswith('vlan'):
                            continue
                        if ifm[arp['localIfIndex']].startswith('irb'):
                            continue
                        is_lldp_int = any(ifm[arp['localIfIndex']] == lldp_int for lldp_int in lldp_ints)
                        if is_lldp_int:
                            continue
                        if self._add_link(
                                prouter=prouter,
                                remote_system_name=self.vrouter_ips[vr_name],
                                local_interface_name=ifm[arp['localIfIndex']],
                                remote_interface_name=vr['if'][-1]['name'],#TODO
                                local_interface_index=arp['localIfIndex'],
                                remote_interface_index=1, #dont know TODO:FIX
                                link_type=2):
                            pass

    def send_uve(self):
        self.uve.send(self.link)

    def switcher(self):
        gevent.sleep(0)

    def scan_data(self):
        t = []
        t.append(gevent.spawn(self.get_vrouters))
        t.append(gevent.spawn(self.get_prouters))
        gevent.joinall(t)

    def _del_uves(self, prouters):
        with self._sem:
            for prouter in prouters:
                self.uve.delete(prouter.name)

    def run(self):
        self._sem = Semaphore()
        self.constnt_schdlr = ConsistentScheduler(
                            self.uve._moduleid,
                            zookeeper=self._config.zookeeper_server(),
                            delete_hndlr=self._del_uves,
                            cluster_id=self._config.cluster_id())

        while self._keep_running:
            self.scan_data()
            if self.constnt_schdlr.schedule(self.prouters):
                try:
                    with self._sem:
                        self.compute()
                        self.send_uve()
                except Exception as e:
                    traceback.print_exc()
                    print str(e)
                gevent.sleep(self._sleep_time)
            else:
                gevent.sleep(1)
        self.constnt_schdlr.finish()
Exemplo n.º 20
0
class Controller(object):
    def __init__(self, config):
        self._config = config
        self._hostname = socket.gethostname()
        self.analytic_api = AnalyticApiClient(self._config)
        self._config.random_collectors = self._config.collectors()
        self._chksum = ""
        self._api_server_checksum = ""
        if self._config.collectors():
            self._chksum = hashlib.md5("".join(self._config.collectors())).hexdigest()
            self._config.random_collectors = random.sample(self._config.collectors(), \
                                                           len(self._config.collectors()))
        if self._config.api_server_list():
            self._api_server_checksum = hashlib.md5("".join(
                self._config.api_server_list())).hexdigest()
            random_api_servers = random.sample(
                self._config.api_server_list(),
                len(self._config.api_server_list()))
            self._config.set_api_server_list(random_api_servers)
        self.uve = LinkUve(self._config)
        self._logger = self.uve.logger()
        self.sleep_time()
        self._keep_running = True
        self._vnc = None
        self._members = None
        self._partitions = None
        self._prouters = {}
        self._vrouter_l2ifs = {}
        self._old_vrouter_l2ifs = {}

    def stop(self):
        self._keep_running = False

    def sleep_time(self, newtime=None):
        if newtime:
            self._sleep_time = newtime
        else:
            self._sleep_time = self._config.frequency()
        return self._sleep_time

    def get_vrouters(self):
        self.analytic_api.get_vrouters(True)
        self.vrouters = {}
        self.vrouter_ips = {}
        self.vrouter_macs = {}
        for vr in self.analytic_api.list_vrouters():
            cfilt = ['VrouterAgent:phy_if', 'VrouterAgent:self_ip_list',
                'VRouterL2IfInfo']
            try:
                d = self.analytic_api.get_vrouter(vr, ','.join(cfilt))
            except Exception as e:
                traceback.print_exc()
                print str(e)
                d = {}
            if 'VrouterAgent' not in d or\
                'self_ip_list' not in d['VrouterAgent'] or\
                'phy_if' not in d['VrouterAgent']:
                continue
            self.vrouters[vr] = {'ips': d['VrouterAgent']['self_ip_list'],
                'if': d['VrouterAgent']['phy_if']
            }
            try:
                self.vrouters[vr]['l2_if'] = d['VRouterL2IfInfo']['if_info']
            except KeyError:
                pass
            for ip in d['VrouterAgent']['self_ip_list']:
                self.vrouter_ips[ip] = vr # index
            for intf in d['VrouterAgent']['phy_if']:
                try:
                    self.vrouter_macs[intf['mac_address']] = {}
                    self.vrouter_macs[intf['mac_address']]['vrname'] = vr
                    self.vrouter_macs[intf['mac_address']]['ifname'] = intf['name']
                except:
                    continue

    def get_prouters(self):
        self.analytic_api.get_prouters(True)
        self.prouters = []
        for pr in self.analytic_api.list_prouters():
            try:
                data = self.analytic_api.get_prouter(pr, 'PRouterEntry')
                if data:
                    self.prouters.append(PRouter(pr, data))
            except Exception as e:
                traceback.print_exc()
                print str(e)

    def _is_linkup(self, prouter, ifindex):
        if 'PRouterEntry' in prouter.data and \
            'ifIndexOperStatusTable' in prouter.data['PRouterEntry']:
                status = filter(lambda x: x['ifIndex'] == ifindex,
                        prouter.data['PRouterEntry']['ifIndexOperStatusTable'])
                if status and status[0]['ifOperStatus'] == 1:
                    return True
        return False

    def _add_link(self, prouter, remote_system_name, local_interface_name,
                  remote_interface_name, local_interface_index,
                  remote_interface_index, link_type):
        # If the remote_system_name or remote_interface_name is None, do not
        # add this link in the link_table.
        if not all([remote_system_name, remote_interface_name]):
            return False
        d = dict(remote_system_name=remote_system_name,
                 local_interface_name=local_interface_name,
                 remote_interface_name=remote_interface_name,
                 local_interface_index=local_interface_index,
                 remote_interface_index=remote_interface_index,
                 type=link_type)
        if link_type == RemoteType.VRouter:
            l2_if = self.vrouters[remote_system_name].get('l2_if')
            if l2_if and remote_interface_name in l2_if:
                if l2_if[remote_interface_name]['remote_system_name'] != \
                        prouter.name:
                    return False
        if self._is_linkup(prouter, local_interface_index):
            if prouter.name in self.link:
                self.link[prouter.name].append(d)
            else:
                self.link[prouter.name] = [d]
            return True
        return False

    def _chk_lnk(self, pre, index):
        if 'ifIndexOperStatusTable' in pre:
            for d in pre['ifIndexOperStatusTable']:
                if d['ifIndex'] == index:
                    return d['ifOperStatus'] == 1
        return False

    def _send_topology_uve(self, members, partitions, prouters):
        topology_info = TopologyInfo()
        if self._members != members:
            self._members = members
            topology_info.members = members
        if self._partitions != partitions:
            self._partitions = partitions
            topology_info.partitions = partitions
        new_prouters = {p.name: p for p in prouters}
        if self._prouters.keys() != new_prouters.keys():
            deleted_prouters = [v for p, v in self._prouters.iteritems() \
                if p not in new_prouters]
            self._del_uves(deleted_prouters)
            self._prouters = new_prouters
            topology_info.prouters = self._prouters.keys()
        if topology_info != TopologyInfo():
            topology_info.name = self._hostname
            TopologyUVE(data=topology_info).send()
    # end _send_topology_uve

    def bms_links(self, prouter, ifm):
        if self._vnc:
            try:
                for li in self._vnc.logical_interfaces_list()[
                            'logical-interfaces']:
                    if prouter.name in li['fq_name']:
                        lif = self._vnc.logical_interface_read(id=li['uuid'])
                        for vmif in lif.get_virtual_machine_interface_refs():
                            vmi = self._vnc.virtual_machine_interface_read(
                                    id=vmif['uuid'])
                            for mc in vmi.virtual_machine_interface_mac_addresses.get_mac_address():
                                ifi = [k for k in ifm if ifm[k] in li[
                                                    'fq_name']][0]
                                rsys = '-'.join(['bms', 'host'] + mc.split(
                                            ':'))
                                if self._add_link(
                                        prouter=prouter,
                                        remote_system_name=rsys,
                                        local_interface_name=li['fq_name'][
                                                                    -1],
                                        remote_interface_name='em0',#no idea
                                        local_interface_index=ifi,
                                        remote_interface_index=1, #dont know TODO:FIX
                                        link_type=RemoteType.BMS):
                                    pass
            except:
                traceback.print_exc()
                self._vnc = None # refresh


    def compute(self):
        self.link = {}
        self._old_vrouter_l2ifs = self._vrouter_l2ifs
        self._vrouter_l2ifs = {}
        for prouter in self.constnt_schdlr.work_items():
            pr, d = prouter.name, prouter.data
            if 'PRouterEntry' not in d or 'ifTable' not in d['PRouterEntry']:
                continue
            self.link[pr] = []
            lldp_ints = []
            ifm = dict(map(lambda x: (x['ifIndex'], x['ifDescr']),
                        d['PRouterEntry']['ifTable']))
            self.bms_links(prouter, ifm)
            for pl in d['PRouterEntry']['lldpTable']['lldpRemoteSystemsData']:
                if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                    'lldpLocSysDesc'].startswith('Cisco'):
                    loc_pname = [x for x in d['PRouterEntry']['lldpTable'][
                            'lldpLocalSystemData']['lldpLocPortTable'] if x[
                            'lldpLocPortNum'] == pl['lldpRemLocalPortNum']][
                            0]['lldpLocPortDesc']
                    pl['lldpRemLocalPortNum'] = [k for k in ifm if ifm[
                                            k] == loc_pname][0]
                elif d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                       'lldpLocSysDesc'].startswith('Arista'):
                       loc_pname = [x for x in d['PRouterEntry']['lldpTable'][
                               'lldpLocalSystemData']['lldpLocPortTable'] if x[
                               'lldpLocPortNum'] == pl['lldpRemLocalPortNum']][
                               0]['lldpLocPortId']
                       pl['lldpRemLocalPortNum'] = [k for k in ifm if ifm[
                                               k] == loc_pname][0]
                if pl['lldpRemLocalPortNum'] in ifm and self._chk_lnk(
                        d['PRouterEntry'], pl['lldpRemLocalPortNum']):
                    if pl['lldpRemPortId'].isdigit():
                        rii = int(pl['lldpRemPortId'])
                    else:
                        try:
                            if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                                  'lldpLocSysDesc'].startswith('Arista'):
                                   rpn = filter(lambda y: y['lldpLocPortId'] == pl[
                                           'lldpRemPortId'], [
                                           x for x in self.prouters if x.name == pl[
                                           'lldpRemSysName']][0].data['PRouterEntry'][
                                           'lldpTable']['lldpLocalSystemData'][
                                           'lldpLocPortTable'])[0]['lldpLocPortId']
                            else:
                                 rpn = filter(lambda y: y['lldpLocPortId'] == pl[
                                  'lldpRemPortId'], [
                                  x for x in self.prouters if x.name == pl[
                                  'lldpRemSysName']][0].data['PRouterEntry'][
                                  'lldpTable']['lldpLocalSystemData'][
                                  'lldpLocPortTable'])[0]['lldpLocPortDesc']
                            rii = filter(lambda y: y['ifDescr'] == rpn,
                                    [ x for x in self.prouters \
                                    if x.name == pl['lldpRemSysName']][0].data[
                                    'PRouterEntry']['ifTable'])[0]['ifIndex']
                        except:
                            rii = 0

                    if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][
                         'lldpLocSysDesc'].startswith('Arista'):
                       if self._add_link(
                            prouter=prouter,
                            remote_system_name=pl['lldpRemSysName'],
                            local_interface_name=ifm[pl['lldpRemLocalPortNum']],
                            remote_interface_name=pl['lldpRemPortId'],
                            local_interface_index=pl['lldpRemLocalPortNum'],
                            remote_interface_index=rii,
                            link_type=RemoteType.PRouter):
                                lldp_ints.append(ifm[pl['lldpRemLocalPortNum']])
                    else:
                         if self._add_link(
                              prouter=prouter,
                              remote_system_name=pl['lldpRemSysName'],
                              local_interface_name=ifm[pl['lldpRemLocalPortNum']],
                              remote_interface_name=pl['lldpRemPortDesc'],
                              local_interface_index=pl['lldpRemLocalPortNum'],
                              remote_interface_index=rii,
                              link_type=RemoteType.PRouter):
                                  lldp_ints.append(ifm[pl['lldpRemLocalPortNum']])

            vrouter_l2ifs = {}
            if 'fdbPortIfIndexTable' in d['PRouterEntry']:
                dot1d2snmp = map (lambda x: (
                            x['dot1dBasePortIfIndex'],
                            x['snmpIfIndex']),
                        d['PRouterEntry']['fdbPortIfIndexTable'])
                dot1d2snmp_dict = dict(dot1d2snmp)
                if 'fdbPortTable' in d['PRouterEntry']:
                    for mac_entry in d['PRouterEntry']['fdbPortTable']:
                        if mac_entry['mac'] in self.vrouter_macs:
                            vrouter_mac_entry = self.vrouter_macs[mac_entry['mac']]
                            vr_name = vrouter_mac_entry['vrname']
                            vr_ifname = vrouter_mac_entry['ifname']
                            fdbport = mac_entry['dot1dBasePortIfIndex']
                            try:
                                snmpport = dot1d2snmp_dict[fdbport]
                                ifname = ifm[snmpport]
                            except:
                                continue
                            is_lldp_int = any(ifname == lldp_int for lldp_int in lldp_ints)
                            if is_lldp_int:
                                continue
                            if self._add_link(
                                    prouter=prouter,
                                    remote_system_name=vr_name,
                                    local_interface_name=ifname,
                                    remote_interface_name=vr_ifname,
                                    local_interface_index=snmpport,
                                    remote_interface_index=1, #dont know TODO:FIX
                                    link_type=RemoteType.VRouter):
                                if vr_name not in vrouter_l2ifs:
                                    vrouter_l2ifs[vr_name] = {}
                                vrouter_l2ifs[vr_name][vr_ifname] = {
                                    'remote_system_name': prouter.name,
                                    'remote_if_name': ifname,
                                }
            for arp in d['PRouterEntry']['arpTable']:
                if arp['ip'] in self.vrouter_ips:
                    if arp['mac'] in map(lambda x: x['mac_address'],
                            self.vrouters[self.vrouter_ips[arp['ip']]]['if']):
                        vr_name = self.vrouter_macs[arp['mac']]['vrname']
                        vr_ifname = self.vrouter_macs[arp['mac']]['ifname']
                        try:
                            if vrouter_l2ifs[vr_name][vr_ifname]\
                                ['remote_system_name'] == prouter.name:
                                del vrouter_l2ifs[vr_name][vr_ifname]
                                if not vrouter_l2ifs[vr_name]:
                                    del vrouter_l2ifs[vr_name]
                                continue
                        except KeyError:
                            pass
                        if ifm[arp['localIfIndex']].startswith('vlan'):
                            continue
                        if ifm[arp['localIfIndex']].startswith('irb'):
                            continue
                        is_lldp_int = any(ifm[arp['localIfIndex']] == lldp_int for lldp_int in lldp_ints)
                        if is_lldp_int:
                            continue
                        if self._add_link(
                                prouter=prouter,
                                remote_system_name=vr_name,
                                local_interface_name=ifm[arp['localIfIndex']],
                                remote_interface_name=vr_ifname,
                                local_interface_index=arp['localIfIndex'],
                                remote_interface_index=1, #dont know TODO:FIX
                                link_type=RemoteType.VRouter):
                            pass
            for vr, intf in vrouter_l2ifs.iteritems():
                if vr in self._vrouter_l2ifs:
                    self._vrouter_l2ifs[vr].update(vrouter_l2ifs[vr])
                else:
                    self._vrouter_l2ifs[vr] = intf

    def send_uve(self):
        old_vrs = set(self._old_vrouter_l2ifs.keys())
        new_vrs = set(self._vrouter_l2ifs.keys())
        del_vrs = old_vrs - new_vrs
        add_vrs = new_vrs - old_vrs
        same_vrs = old_vrs.intersection(new_vrs)
        for vr in del_vrs:
            vr_l2info = VRouterL2IfInfo(name=vr, deleted=True)
            VRouterL2IfUVE(data=vr_l2info).send()
        for vr in add_vrs:
            if_info = {}
            for vrif, remif_info in self._vrouter_l2ifs[vr].iteritems():
                if_info[vrif] = RemoteIfInfo(remif_info['remote_system_name'],
                    remif_info['remote_if_name'])
            vr_l2info = VRouterL2IfInfo(name=vr, if_info=if_info)
            VRouterL2IfUVE(data=vr_l2info).send()
        for vr in same_vrs:
            if self._vrouter_l2ifs[vr] != self._old_vrouter_l2ifs[vr]:
                if_info = {}
                for vrif, remif_info in self._vrouter_l2ifs[vr].iteritems():
                    if_info[vrif] = RemoteIfInfo(
                        remif_info['remote_system_name'],
                        remif_info['remote_if_name'])
                vr_l2info = VRouterL2IfInfo(name=vr, if_info=if_info)
                VRouterL2IfUVE(data=vr_l2info).send()
        self.uve.send(self.link)

    def switcher(self):
        gevent.sleep(0)

    def scan_data(self):
        t = []
        t.append(gevent.spawn(self.get_vrouters))
        t.append(gevent.spawn(self.get_prouters))
        gevent.joinall(t)

    def _del_uves(self, prouters):
        with self._sem:
            for prouter in prouters:
                self.uve.delete(prouter.name)

    def sighup_handler(self):
        if self._config._args.conf_file:
            config = ConfigParser.SafeConfigParser()
            config.read(self._config._args.conf_file)
            if 'DEFAULTS' in config.sections():
                try:
                    collectors = config.get('DEFAULTS', 'collectors')
                    if type(collectors) is str:
                        collectors = collectors.split()
                        new_chksum = hashlib.md5("".join(collectors)).hexdigest()
                        if new_chksum != self._chksum:
                            self._chksum = new_chksum
                            self._config.random_collectors = \
                                random.sample(collectors, len(collectors))
                        # Reconnect to achieve load-balance irrespective of list
                        self.uve.sandesh_reconfig_collectors(
                                self._config.random_collectors)
                except ConfigParser.NoOptionError as e:
                    pass
            if 'API_SERVER' in config.sections():
                try:
                    api_servers = config.get('API_SERVER', 'api_server_list')
                except ConfigParser.NoOptionError:
                    pass
                else:
                    if isinstance(api_servers, basestring):
                        api_servers = api_servers.split()
                    new_api_server_checksum = hashlib.md5("".join(
                        api_servers)).hexdigest()
                    if new_api_server_checksum != self._api_server_checksum:
                        self._api_server_checksum = new_api_server_checksum
                        random_api_servers = random.sample(api_servers,
                            len(api_servers))
                        self._config.set_api_server_list(random_api_servers)
                        self._vnc = None
    # end sighup_handler
 
        
    def run(self):

        """ @sighup
        SIGHUP handler to indicate configuration changes 
        """
        gevent.signal(signal.SIGHUP, self.sighup_handler)

        self._sem = Semaphore()
        self.constnt_schdlr = ConsistentScheduler(
                            self.uve._moduleid,
                            zookeeper=self._config.zookeeper_server(),
                            delete_hndlr=self._del_uves,
                            logger=self._logger,
                            cluster_id=self._config.cluster_id())

        while self._keep_running:
            if not self._vnc:
                self._vnc = self._config.vnc_api()
            self.scan_data()
            if self.constnt_schdlr.schedule(self.prouters):
                members = self.constnt_schdlr.members()
                partitions = self.constnt_schdlr.partitions()
                self._send_topology_uve(members, partitions,
                    self.constnt_schdlr.work_items())
                try:
                    with self._sem:
                        self.compute()
                        self.send_uve()
                except Exception as e:
                    traceback.print_exc()
                    print str(e)
                gevent.sleep(self._sleep_time)
            else:
                gevent.sleep(1)
        self.constnt_schdlr.finish()