def run(self): i = 0 self._sem = Semaphore() self._logger.debug('Starting.. %s' % str( self._config.zookeeper_server())) constnt_schdlr = ConsistentScheduler( self._config._name, zookeeper=self._config.zookeeper_server(), delete_hndlr=self._del_uves, logger=self._logger, cluster_id=self._config.cluster_id()) while self._keep_running: self._logger.debug('@run: ittr(%d)' % i) if constnt_schdlr.schedule(self._config.devices()): members = constnt_schdlr.members() partitions = constnt_schdlr.partitions() prouters = map(lambda x: x.name, constnt_schdlr.work_items()) self._send_snmp_collector_uve(members, partitions, prouters) sleep_time = self.do_work(i, constnt_schdlr.work_items()) self._logger.debug('done work %s' % str(prouters)) i += 1 gevent.sleep(sleep_time) else: gevent.sleep(1) constnt_schdlr.finish()
def run(self): """ @sighup SIGHUP handler to indicate configuration changes """ gevent.signal(signal.SIGHUP, self.sighup_handler) i = 0 self._sem = Semaphore() self._logger.debug('Starting.. %s' % str(self._config.zookeeper_server())) constnt_schdlr = ConsistentScheduler( self._config._name, zookeeper=self._config.zookeeper_server(), delete_hndlr=self._del_uves, logger=self._logger, cluster_id=self._config.cluster_id()) while self._keep_running: self._logger.debug('@run: ittr(%d)' % i) if constnt_schdlr.schedule(self._config.devices()): members = constnt_schdlr.members() partitions = constnt_schdlr.partitions() self._send_snmp_collector_uve(members, partitions, constnt_schdlr.work_items()) sleep_time = self.do_work(i, constnt_schdlr.work_items()) self._logger.debug('done work %s' % str(self._prouters.keys())) i += 1 gevent.sleep(sleep_time) else: gevent.sleep(1) constnt_schdlr.finish()
def run(self): """ @sighup SIGHUP handler to indicate configuration changes """ gevent.signal(signal.SIGHUP, self.sighup_handler) i = 0 self._sem = Semaphore() self._logger.debug('Starting.. %s' % str( self._config.zookeeper_server())) constnt_schdlr = ConsistentScheduler( self._config._name, zookeeper=self._config.zookeeper_server(), delete_hndlr=self._del_uves, logger=self._logger, cluster_id=self._config.cluster_id()) while self._keep_running: self._logger.debug('@run: ittr(%d)' % i) if constnt_schdlr.schedule(self._config.devices()): members = constnt_schdlr.members() partitions = constnt_schdlr.partitions() self._send_snmp_collector_uve(members, partitions, constnt_schdlr.work_items()) sleep_time = self.do_work(i, constnt_schdlr.work_items()) self._logger.debug('done work %s' % str(self._prouters.keys())) i += 1 gevent.sleep(sleep_time) else: gevent.sleep(1) constnt_schdlr.finish()
def run(self): i = 0 self._sem = Semaphore() constnt_schdlr = ConsistentScheduler( self._config._name, zookeeper=self._config.zookeeper_server(), delete_hndlr=self._del_uves, logger=self._logger) while self._keep_running: self._logger.debug('@run: ittr(%d)' % i) if constnt_schdlr.schedule(self._config.devices()): sleep_time = self.do_work(i, constnt_schdlr.work_items()) self._logger.debug('done work %s' % str( map(lambda x: x.name, constnt_schdlr.work_items()))) i += 1 gevent.sleep(sleep_time) else: gevent.sleep(1) constnt_schdlr.finish()
class Controller(object): def __init__(self, config): self._config = config if 'host_ip' in self._config._args: host_ip = self._config._args.host_ip else: host_ip = socket.gethostbyname(socket.getfqdn()) self._hostname = socket.getfqdn(host_ip) self.analytic_api = AnalyticApiClient(self._config) self._config.random_collectors = self._config.collectors() self._chksum = "" if self._config.collectors(): self._chksum = hashlib.md5("".join( self._config.collectors())).hexdigest() self._config.random_collectors = random.sample(self._config.collectors(), \ len(self._config.collectors())) self.uve = LinkUve(self._config) self._sandesh = self.uve.sandesh_instance() self._logger = self.uve.logger() self.sleep_time() self._sem = Semaphore() self._members = None self._partitions = None self._prouters = {} self._vrouter_l2ifs = {} self._old_vrouter_l2ifs = {} self._config_handler = TopologyConfigHandler( self._sandesh, self._config.rabbitmq_params(), self._config.cassandra_params(), host_ip) self.constnt_schdlr = ConsistentScheduler( self.uve._moduleid, zookeeper=self._config.zookeeper_server(), delete_hndlr=self._del_uves, logger=self._logger, cluster_id=self._config.cluster_id()) def sleep_time(self, newtime=None): if newtime: self._sleep_time = newtime else: self._sleep_time = self._config.frequency() return self._sleep_time def get_vrouters(self): self.analytic_api.get_vrouters(True) self.vrouters = {} self.vrouter_ips = {} self.vrouter_macs = {} for vr in self.analytic_api.list_vrouters(): cfilt = [ 'VrouterAgent:phy_if', 'VrouterAgent:self_ip_list', 'VRouterL2IfInfo' ] try: d = self.analytic_api.get_vrouter(vr, ','.join(cfilt)) except Exception as e: traceback.print_exc() print(str(e)) d = {} if 'VrouterAgent' not in d or\ 'self_ip_list' not in d['VrouterAgent'] or\ 'phy_if' not in d['VrouterAgent']: continue self.vrouters[vr] = { 'ips': d['VrouterAgent']['self_ip_list'], 'if': d['VrouterAgent']['phy_if'] } try: self.vrouters[vr]['l2_if'] = d['VRouterL2IfInfo']['if_info'] except KeyError: pass for ip in d['VrouterAgent']['self_ip_list']: self.vrouter_ips[ip] = vr # index for intf in d['VrouterAgent']['phy_if']: try: self.vrouter_macs[intf['mac_address']] = {} self.vrouter_macs[intf['mac_address']]['vrname'] = vr self.vrouter_macs[ intf['mac_address']]['ifname'] = intf['name'] except: continue def get_prouters(self): self.analytic_api.get_prouters(True) self.prouters = [] for pr in self.analytic_api.list_prouters(): try: data = self.analytic_api.get_prouter(pr, 'PRouterEntry') if data: self.prouters.append(PRouter(pr, data)) except Exception as e: traceback.print_exc() print(str(e)) def _is_linkup(self, prouter, ifindex): if 'PRouterEntry' in prouter.data and \ 'ifIndexOperStatusTable' in prouter.data['PRouterEntry']: status = [ x for x in prouter.data['PRouterEntry']['ifIndexOperStatusTable'] if x['ifIndex'] == ifindex ] if status and status[0]['ifOperStatus'] == 1: return True return False def _add_link(self, prouter, remote_system_name, local_interface_name, remote_interface_name, local_interface_index, remote_interface_index, link_type): # If the remote_system_name or remote_interface_name is None, do not # add this link in the link_table. if not all([remote_system_name, remote_interface_name]): return False d = dict(remote_system_name=remote_system_name, local_interface_name=local_interface_name, remote_interface_name=remote_interface_name, local_interface_index=local_interface_index, remote_interface_index=remote_interface_index, type=link_type) if link_type == RemoteType.VRouter: l2_if = self.vrouters[remote_system_name].get('l2_if') if l2_if and remote_interface_name in l2_if: if l2_if[remote_interface_name]['remote_system_name'] != \ prouter.name: return False if self._is_linkup(prouter, local_interface_index): if prouter.name in self.link: self.link[prouter.name].append(d) else: self.link[prouter.name] = [d] return True return False def _chk_lnk(self, pre, index): if 'ifIndexOperStatusTable' in pre: for d in pre['ifIndexOperStatusTable']: if d['ifIndex'] == index: return d['ifOperStatus'] == 1 return False def _send_topology_uve(self, members, partitions, prouters): topology_info = TopologyInfo() if self._members != members: self._members = members topology_info.members = members if self._partitions != partitions: self._partitions = partitions topology_info.partitions = partitions new_prouters = {p.name: p for p in prouters} if list(self._prouters.keys()) != list(new_prouters.keys()): deleted_prouters = [v for p, v in self._prouters.items() \ if p not in new_prouters] self._del_uves(deleted_prouters) self._prouters = new_prouters topology_info.prouters = list(self._prouters.keys()) if topology_info != TopologyInfo(): topology_info.name = self._hostname TopologyUVE(data=topology_info).send() # end _send_topology_uve def bms_links(self, prouter, ifm): try: lifs = self._config_handler.get_logical_interfaces() if not lifs: return for lif_fqname, lif in lifs: if prouter.name in lif_fqname: vmi_refs = lif.obj.get_virtual_machine_interface_refs() if not vmi_refs: continue for vmif in vmi_refs: vmi = self._config_handler.\ get_virtual_machine_interface(fq_name=None, uuid=vmif['uuid']) if not vmi: continue vmi = vmi.obj macs = vmi.virtual_machine_interface_mac_addresses.\ get_mac_address() if not macs: continue for mc in macs: ifi = [k for k in ifm if ifm[k] in lif_fqname][0] rsys = '-'.join(['bms', 'host'] + mc.split(':')) self._add_link( prouter=prouter, remote_system_name=rsys, local_interface_name=lif.obj.fq_name[-1], remote_interface_name='em0', #no idea local_interface_index=ifi, remote_interface_index=1, #dont know TODO:FIX link_type=RemoteType.BMS) except: traceback.print_exc() def compute(self): self.link = {} self._old_vrouter_l2ifs = self._vrouter_l2ifs self._vrouter_l2ifs = {} for prouter in self.constnt_schdlr.work_items(): pr, d = prouter.name, prouter.data if 'PRouterEntry' not in d or 'ifTable' not in d['PRouterEntry']: continue self.link[pr] = [] lldp_ints = [] ifm = dict([(x['ifIndex'], x['ifDescr']) for x in d['PRouterEntry']['ifTable']]) self.bms_links(prouter, ifm) for pl in d['PRouterEntry']['lldpTable']['lldpRemoteSystemsData']: if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Cisco'): loc_pname = [ x for x in d['PRouterEntry']['lldpTable'] ['lldpLocalSystemData']['lldpLocPortTable'] if x['lldpLocPortNum'] == pl['lldpRemLocalPortNum'] ][0]['lldpLocPortDesc'] pl['lldpRemLocalPortNum'] = [ k for k in ifm if ifm[k] == loc_pname ][0] elif d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Arista'): loc_pname = [ x for x in d['PRouterEntry']['lldpTable'] ['lldpLocalSystemData']['lldpLocPortTable'] if x['lldpLocPortNum'] == pl['lldpRemLocalPortNum'] ][0]['lldpLocPortId'] pl['lldpRemLocalPortNum'] = [ k for k in ifm if ifm[k] == loc_pname ][0] if pl['lldpRemLocalPortNum'] in ifm and self._chk_lnk( d['PRouterEntry'], pl['lldpRemLocalPortNum']): if pl['lldpRemPortId'].isdigit(): rii = int(pl['lldpRemPortId']) else: try: if d['PRouterEntry']['lldpTable'][ 'lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Arista'): rpn = filter( lambda y: y['lldpLocPortId'] == pl[ 'lldpRemPortId'], [ x for x in self.prouters if x.name == pl['lldpRemSysName'] ][0].data['PRouterEntry']['lldpTable'] ['lldpLocalSystemData'] ['lldpLocPortTable'])[0]['lldpLocPortId'] else: rpn = filter( lambda y: y['lldpLocPortId'] == pl[ 'lldpRemPortId'], [ x for x in self.prouters if x.name == pl['lldpRemSysName'] ][0].data['PRouterEntry']['lldpTable'] ['lldpLocalSystemData'] ['lldpLocPortTable'])[0]['lldpLocPortDesc'] rii = filter(lambda y: y['ifDescr'] == rpn, [ x for x in self.prouters \ if x.name == pl['lldpRemSysName']][0].data[ 'PRouterEntry']['ifTable'])[0]['ifIndex'] except: rii = 0 if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Arista'): if self._add_link( prouter=prouter, remote_system_name=pl['lldpRemSysName'], local_interface_name=ifm[ pl['lldpRemLocalPortNum']], remote_interface_name=pl['lldpRemPortId'], local_interface_index=pl[ 'lldpRemLocalPortNum'], remote_interface_index=rii, link_type=RemoteType.PRouter): lldp_ints.append(ifm[pl['lldpRemLocalPortNum']]) else: if self._add_link( prouter=prouter, remote_system_name=pl['lldpRemSysName'], local_interface_name=ifm[ pl['lldpRemLocalPortNum']], remote_interface_name=pl['lldpRemPortDesc'], local_interface_index=pl[ 'lldpRemLocalPortNum'], remote_interface_index=rii, link_type=RemoteType.PRouter): lldp_ints.append(ifm[pl['lldpRemLocalPortNum']]) vrouter_l2ifs = {} if 'fdbPortIfIndexTable' in d['PRouterEntry']: dot1d2snmp = [(x['dot1dBasePortIfIndex'], x['snmpIfIndex']) for x in d['PRouterEntry']['fdbPortIfIndexTable'] ] dot1d2snmp_dict = dict(dot1d2snmp) if 'fdbPortTable' in d['PRouterEntry']: for mac_entry in d['PRouterEntry']['fdbPortTable']: if mac_entry['mac'] in self.vrouter_macs: vrouter_mac_entry = self.vrouter_macs[ mac_entry['mac']] vr_name = vrouter_mac_entry['vrname'] vr_ifname = vrouter_mac_entry['ifname'] fdbport = mac_entry['dot1dBasePortIfIndex'] try: snmpport = dot1d2snmp_dict[fdbport] ifname = ifm[snmpport] except: continue is_lldp_int = any(ifname == lldp_int for lldp_int in lldp_ints) if is_lldp_int: continue if self._add_link( prouter=prouter, remote_system_name=vr_name, local_interface_name=ifname, remote_interface_name=vr_ifname, local_interface_index=snmpport, remote_interface_index= 1, #dont know TODO:FIX link_type=RemoteType.VRouter): if vr_name not in vrouter_l2ifs: vrouter_l2ifs[vr_name] = {} vrouter_l2ifs[vr_name][vr_ifname] = { 'remote_system_name': prouter.name, 'remote_if_name': ifname, } for arp in d['PRouterEntry']['arpTable']: if arp['ip'] in self.vrouter_ips: if arp['mac'] in [ x['mac_address'] for x in self.vrouters[ self.vrouter_ips[arp['ip']]]['if'] ]: vr_name = self.vrouter_macs[arp['mac']]['vrname'] vr_ifname = self.vrouter_macs[arp['mac']]['ifname'] try: if vrouter_l2ifs[vr_name][vr_ifname]\ ['remote_system_name'] == prouter.name: del vrouter_l2ifs[vr_name][vr_ifname] if not vrouter_l2ifs[vr_name]: del vrouter_l2ifs[vr_name] continue except KeyError: pass if ifm[arp['localIfIndex']].startswith('vlan'): continue if ifm[arp['localIfIndex']].startswith('irb'): continue is_lldp_int = any(ifm[arp['localIfIndex']] == lldp_int for lldp_int in lldp_ints) if is_lldp_int: continue if self._add_link( prouter=prouter, remote_system_name=vr_name, local_interface_name=ifm[arp['localIfIndex']], remote_interface_name=vr_ifname, local_interface_index=arp['localIfIndex'], remote_interface_index=1, #dont know TODO:FIX link_type=RemoteType.VRouter): pass for vr, intf in vrouter_l2ifs.items(): if vr in self._vrouter_l2ifs: self._vrouter_l2ifs[vr].update(vrouter_l2ifs[vr]) else: self._vrouter_l2ifs[vr] = intf def send_uve(self): old_vrs = set(self._old_vrouter_l2ifs.keys()) new_vrs = set(self._vrouter_l2ifs.keys()) del_vrs = old_vrs - new_vrs add_vrs = new_vrs - old_vrs same_vrs = old_vrs.intersection(new_vrs) for vr in del_vrs: vr_l2info = VRouterL2IfInfo(name=vr, deleted=True) VRouterL2IfUVE(data=vr_l2info).send() for vr in add_vrs: if_info = {} for vrif, remif_info in self._vrouter_l2ifs[vr].items(): if_info[vrif] = RemoteIfInfo(remif_info['remote_system_name'], remif_info['remote_if_name']) vr_l2info = VRouterL2IfInfo(name=vr, if_info=if_info) VRouterL2IfUVE(data=vr_l2info).send() for vr in same_vrs: if self._vrouter_l2ifs[vr] != self._old_vrouter_l2ifs[vr]: if_info = {} for vrif, remif_info in self._vrouter_l2ifs[vr].items(): if_info[vrif] = RemoteIfInfo( remif_info['remote_system_name'], remif_info['remote_if_name']) vr_l2info = VRouterL2IfInfo(name=vr, if_info=if_info) VRouterL2IfUVE(data=vr_l2info).send() self.uve.send(self.link) def switcher(self): gevent.sleep(0) def scan_data(self): t = [] t.append(gevent.spawn(self.get_vrouters)) t.append(gevent.spawn(self.get_prouters)) gevent.joinall(t) def _del_uves(self, prouters): with self._sem: for prouter in prouters: self.uve.delete(prouter.name) def sighup_handler(self): if self._config._args.conf_file: config = configparser.SafeConfigParser() config.read(self._config._args.conf_file) if 'DEFAULTS' in config.sections(): try: collectors = config.get('DEFAULTS', 'collectors') if isinstance(collectors, (basestring, str)): collectors = collectors.split() new_chksum = hashlib.md5( "".join(collectors)).hexdigest() if new_chksum != self._chksum: self._chksum = new_chksum self._config.random_collectors = \ random.sample(collectors, len(collectors)) # Reconnect to achieve load-balance irrespective of list self.uve.sandesh_reconfig_collectors( self._config.random_collectors) except configparser.NoOptionError as e: pass # end sighup_handler def _uve_scanner(self): while True: self.scan_data() if self.constnt_schdlr.schedule(self.prouters): members = self.constnt_schdlr.members() partitions = self.constnt_schdlr.partitions() self._send_topology_uve(members, partitions, self.constnt_schdlr.work_items()) try: with self._sem: self.compute() self.send_uve() except Exception as e: traceback.print_exc() print(str(e)) gevent.sleep(self._sleep_time) else: gevent.sleep(1) # end _uve_scanner def run(self): """ @sighup SIGHUP handler to indicate configuration changes """ gevent.signal(signal.SIGHUP, self.sighup_handler) self.gevs = [ gevent.spawn(self._config_handler.start), gevent.spawn(self._uve_scanner) ] try: gevent.joinall(self.gevs) except KeyboardInterrupt: self._logger.error('Exiting on ^C') except gevent.GreenletExit: self._logger.error('Exiting on gevent-kill') finally: self._logger.error('stopping everything!') self.stop() # end run def stop(self): self.uve.stop() l = len(self.gevs) for i in range(0, l): self._logger.error('killing %d of %d' % (i + 1, l)) self.gevs[0].kill() self._logger.error('joining %d of %d' % (i + 1, l)) self.gevs[0].join() self._logger.error('stopped %d of %d' % (i + 1, l)) self.gevs.pop(0) self.constnt_schdlr.finish()
class Controller(object): def __init__(self, config): self._config = config self._hostname = socket.gethostname() self.analytic_api = AnalyticApiClient(self._config) self._config.random_collectors = self._config.collectors() self._chksum = "" self._api_server_checksum = "" if self._config.collectors(): self._chksum = hashlib.md5("".join( self._config.collectors())).hexdigest() self._config.random_collectors = random.sample(self._config.collectors(), \ len(self._config.collectors())) if self._config.api_server_list(): self._api_server_checksum = hashlib.md5("".join( self._config.api_server_list())).hexdigest() random_api_servers = random.sample( self._config.api_server_list(), len(self._config.api_server_list())) self._config.set_api_server_list(random_api_servers) self.uve = LinkUve(self._config) self._logger = self.uve.logger() self.sleep_time() self._keep_running = True self._vnc = None self._members = None self._partitions = None self._prouters = None def stop(self): self._keep_running = False def sleep_time(self, newtime=None): if newtime: self._sleep_time = newtime else: self._sleep_time = self._config.frequency() return self._sleep_time def get_vrouters(self): self.analytic_api.get_vrouters(True) self.vrouters = {} self.vrouter_ips = {} self.vrouter_macs = {} for vr in self.analytic_api.list_vrouters(): try: d = self.analytic_api.get_vrouter(vr, 'VrouterAgent:phy_if') except Exception as e: traceback.print_exc() print str(e) d = {} if 'VrouterAgent' not in d: d['VrouterAgent'] = {} try: _ipl = self.analytic_api.get_vrouter( vr, 'VrouterAgent:self_ip_list') except Exception as e: traceback.print_exc() print str(e) _ipl = {} if 'VrouterAgent' in _ipl: d['VrouterAgent'].update(_ipl['VrouterAgent']) if 'VrouterAgent' not in d or\ 'self_ip_list' not in d['VrouterAgent'] or\ 'phy_if' not in d['VrouterAgent']: continue self.vrouters[vr] = { 'ips': d['VrouterAgent']['self_ip_list'], 'if': d['VrouterAgent']['phy_if'], } for ip in d['VrouterAgent']['self_ip_list']: self.vrouter_ips[ip] = vr # index for intf in d['VrouterAgent']['phy_if']: try: self.vrouter_macs[intf['mac_address']] = {} self.vrouter_macs[intf['mac_address']]['vrname'] = vr self.vrouter_macs[ intf['mac_address']]['ifname'] = intf['name'] except: continue def get_prouters(self): self.analytic_api.get_prouters(True) self.prouters = [] for pr in self.analytic_api.list_prouters(): try: self.prouters.append( PRouter(pr, self.analytic_api.get_prouter(pr, 'PRouterEntry'))) except Exception as e: traceback.print_exc() print str(e) def _is_linkup(self, prouter, ifindex): if 'PRouterEntry' in prouter.data and \ 'ifIndexOperStatusTable' in prouter.data['PRouterEntry']: status = filter( lambda x: x['ifIndex'] == ifindex, prouter.data['PRouterEntry']['ifIndexOperStatusTable']) if status and status[0]['ifOperStatus'] == 1: return True return False def _add_link(self, prouter, remote_system_name, local_interface_name, remote_interface_name, local_interface_index, remote_interface_index, link_type): d = dict(remote_system_name=remote_system_name, local_interface_name=local_interface_name, remote_interface_name=remote_interface_name, local_interface_index=local_interface_index, remote_interface_index=remote_interface_index, type=link_type) if self._is_linkup(prouter, local_interface_index): if prouter.name in self.link: self.link[prouter.name].append(d) else: self.link[prouter.name] = [d] return True return False def _chk_lnk(self, pre, index): if 'ifIndexOperStatusTable' in pre: for d in pre['ifIndexOperStatusTable']: if d['ifIndex'] == index: return d['ifOperStatus'] == 1 return False def _send_topology_uve(self, members, partitions, prouters): topology_info = TopologyInfo() if self._members != members: self._members = members topology_info.members = members if self._partitions != partitions: self._partitions = partitions topology_info.partitions = partitions if self._prouters != prouters: self._prouters = prouters topology_info.prouters = prouters if topology_info != TopologyInfo(): topology_info.name = self._hostname TopologyUVE(data=topology_info).send() # end _send_topology_uve def bms_links(self, prouter, ifm): if self._vnc: try: for li in self._vnc.logical_interfaces_list( )['logical-interfaces']: if prouter.name in li['fq_name']: lif = self._vnc.logical_interface_read(id=li['uuid']) for vmif in lif.get_virtual_machine_interface_refs(): vmi = self._vnc.virtual_machine_interface_read( id=vmif['uuid']) for mc in vmi.virtual_machine_interface_mac_addresses.get_mac_address( ): ifi = [ k for k in ifm if ifm[k] in li['fq_name'] ][0] rsys = '-'.join(['bms', 'host'] + mc.split(':')) if self._add_link( prouter=prouter, remote_system_name=rsys, local_interface_name=li['fq_name'][-1], remote_interface_name='em0', #no idea local_interface_index=ifi, remote_interface_index= 1, #dont know TODO:FIX link_type=2): pass except: traceback.print_exc() self._vnc = None # refresh def compute(self): self.link = {} for prouter in self.constnt_schdlr.work_items(): pr, d = prouter.name, prouter.data if 'PRouterEntry' not in d or 'ifTable' not in d['PRouterEntry']: continue self.link[pr] = [] lldp_ints = [] ifm = dict( map(lambda x: (x['ifIndex'], x['ifDescr']), d['PRouterEntry']['ifTable'])) self.bms_links(prouter, ifm) for pl in d['PRouterEntry']['lldpTable']['lldpRemoteSystemsData']: if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Cisco'): loc_pname = [ x for x in d['PRouterEntry']['lldpTable'] ['lldpLocalSystemData']['lldpLocPortTable'] if x['lldpLocPortNum'] == pl['lldpRemLocalPortNum'] ][0]['lldpLocPortDesc'] pl['lldpRemLocalPortNum'] = [ k for k in ifm if ifm[k] == loc_pname ][0] elif d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Arista'): loc_pname = [ x for x in d['PRouterEntry']['lldpTable'] ['lldpLocalSystemData']['lldpLocPortTable'] if x['lldpLocPortNum'] == pl['lldpRemLocalPortNum'] ][0]['lldpLocPortId'] pl['lldpRemLocalPortNum'] = [ k for k in ifm if ifm[k] == loc_pname ][0] if pl['lldpRemLocalPortNum'] in ifm and self._chk_lnk( d['PRouterEntry'], pl['lldpRemLocalPortNum']): if pl['lldpRemPortId'].isdigit(): rii = int(pl['lldpRemPortId']) else: try: if d['PRouterEntry']['lldpTable'][ 'lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Arista'): rpn = filter( lambda y: y['lldpLocPortId'] == pl[ 'lldpRemPortId'], [ x for x in self.prouters if x.name == pl['lldpRemSysName'] ][0].data['PRouterEntry']['lldpTable'] ['lldpLocalSystemData'] ['lldpLocPortTable'])[0]['lldpLocPortId'] else: rpn = filter( lambda y: y['lldpLocPortId'] == pl[ 'lldpRemPortId'], [ x for x in self.prouters if x.name == pl['lldpRemSysName'] ][0].data['PRouterEntry']['lldpTable'] ['lldpLocalSystemData'] ['lldpLocPortTable'])[0]['lldpLocPortDesc'] rii = filter(lambda y: y['ifDescr'] == rpn, [ x for x in self.prouters \ if x.name == pl['lldpRemSysName']][0].data[ 'PRouterEntry']['ifTable'])[0]['ifIndex'] except: rii = 0 if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Arista'): if self._add_link( prouter=prouter, remote_system_name=pl['lldpRemSysName'], local_interface_name=ifm[ pl['lldpRemLocalPortNum']], remote_interface_name=pl['lldpRemPortId'], local_interface_index=pl[ 'lldpRemLocalPortNum'], remote_interface_index=rii, link_type=1): lldp_ints.append(ifm[pl['lldpRemLocalPortNum']]) else: if self._add_link( prouter=prouter, remote_system_name=pl['lldpRemSysName'], local_interface_name=ifm[ pl['lldpRemLocalPortNum']], remote_interface_name=pl['lldpRemPortDesc'], local_interface_index=pl[ 'lldpRemLocalPortNum'], remote_interface_index=rii, link_type=1): lldp_ints.append(ifm[pl['lldpRemLocalPortNum']]) vrouter_neighbors = [] if 'fdbPortIfIndexTable' in d['PRouterEntry']: dot1d2snmp = map( lambda x: (x['dot1dBasePortIfIndex'], x['snmpIfIndex']), d['PRouterEntry']['fdbPortIfIndexTable']) dot1d2snmp_dict = dict(dot1d2snmp) if 'fdbPortTable' in d['PRouterEntry']: for mac_entry in d['PRouterEntry']['fdbPortTable']: if mac_entry['mac'] in self.vrouter_macs: vrouter_mac_entry = self.vrouter_macs[ mac_entry['mac']] fdbport = mac_entry['dot1dBasePortIfIndex'] try: snmpport = dot1d2snmp_dict[fdbport] ifname = ifm[snmpport] except: continue is_lldp_int = any(ifname == lldp_int for lldp_int in lldp_ints) if is_lldp_int: continue if self._add_link( prouter=prouter, remote_system_name=vrouter_mac_entry[ 'vrname'], local_interface_name=ifname, remote_interface_name=vrouter_mac_entry[ 'ifname'], local_interface_index=snmpport, remote_interface_index= 1, #dont know TODO:FIX link_type=2): vrouter_neighbors.append( vrouter_mac_entry['vrname']) for arp in d['PRouterEntry']['arpTable']: if arp['ip'] in self.vrouter_ips: if arp['mac'] in map( lambda x: x['mac_address'], self.vrouters[self.vrouter_ips[arp['ip']]]['if']): vr_name = arp['ip'] vr = self.vrouters[self.vrouter_ips[vr_name]] if self.vrouter_ips[vr_name] in vrouter_neighbors: continue if ifm[arp['localIfIndex']].startswith('vlan'): continue if ifm[arp['localIfIndex']].startswith('irb'): continue is_lldp_int = any(ifm[arp['localIfIndex']] == lldp_int for lldp_int in lldp_ints) if is_lldp_int: continue if self._add_link( prouter=prouter, remote_system_name=self.vrouter_ips[vr_name], local_interface_name=ifm[arp['localIfIndex']], remote_interface_name=vr['if'][-1] ['name'], #TODO local_interface_index=arp['localIfIndex'], remote_interface_index=1, #dont know TODO:FIX link_type=2): pass def send_uve(self): self.uve.send(self.link) def switcher(self): gevent.sleep(0) def scan_data(self): t = [] t.append(gevent.spawn(self.get_vrouters)) t.append(gevent.spawn(self.get_prouters)) gevent.joinall(t) def _del_uves(self, prouters): with self._sem: for prouter in prouters: self.uve.delete(prouter.name) def sighup_handler(self): if self._config._args.conf_file: config = ConfigParser.SafeConfigParser() config.read(self._config._args.conf_file) if 'DEFAULTS' in config.sections(): try: collectors = config.get('DEFAULTS', 'collectors') if type(collectors) is str: collectors = collectors.split() new_chksum = hashlib.md5( "".join(collectors)).hexdigest() if new_chksum != self._chksum: self._chksum = new_chksum self._config.random_collectors = \ random.sample(collectors, len(collectors)) # Reconnect to achieve load-balance irrespective of list self.uve.sandesh_reconfig_collectors( self._config.random_collectors) except ConfigParser.NoOptionError as e: pass if 'API_SERVER' in config.sections(): try: api_servers = config.get('API_SERVER', 'api_server_list') except ConfigParser.NoOptionError: pass else: if isinstance(api_servers, basestring): api_servers = api_servers.split() new_api_server_checksum = hashlib.md5( "".join(api_servers)).hexdigest() if new_api_server_checksum != self._api_server_checksum: self._api_server_checksum = new_api_server_checksum random_api_servers = random.sample( api_servers, len(api_servers)) self._config.set_api_server_list(random_api_servers) self._vnc = None # end sighup_handler def run(self): """ @sighup SIGHUP handler to indicate configuration changes """ gevent.signal(signal.SIGHUP, self.sighup_handler) self._sem = Semaphore() self.constnt_schdlr = ConsistentScheduler( self.uve._moduleid, zookeeper=self._config.zookeeper_server(), delete_hndlr=self._del_uves, logger=self._logger, cluster_id=self._config.cluster_id()) while self._keep_running: if not self._vnc: self._vnc = self._config.vnc_api() self.scan_data() if self.constnt_schdlr.schedule(self.prouters): members = self.constnt_schdlr.members() partitions = self.constnt_schdlr.partitions() prouters = map(lambda x: x.name, self.constnt_schdlr.work_items()) self._send_topology_uve(members, partitions, prouters) try: with self._sem: self.compute() self.send_uve() except Exception as e: traceback.print_exc() print str(e) gevent.sleep(self._sleep_time) else: gevent.sleep(1) self.constnt_schdlr.finish()
class Controller(object): def __init__(self, config): self._config = config self._config.random_collectors = self._config.collectors() self._chksum = "" if self._config.collectors(): self._chksum = hashlib.md5("".join(self._config.collectors())).hexdigest() self._config.random_collectors = random.sample(self._config.collectors(), \ len(self._config.collectors())) self.uve = SnmpUve(self._config) self._sandesh = self.uve.sandesh_instance() self._hostname = socket.gethostname() self._logger = self.uve.logger() self.sleep_time() self.last = set() self._sem = Semaphore() self._config.set_cb(self.notify) self._mnt = MaxNinTtime(3, self._sleep_time) self._state = 'full_scan' # replace it w/ fsm self._if_data = None # replace it w/ fsm self._cleanup = None self._members = None self._partitions = None self._prouters = {} self._config_handler = SnmpConfigHandler(self._sandesh, self._config.rabbitmq_params(), self._config.cassandra_params()) self._consistent_scheduler = ConsistentScheduler(self._config._name, zookeeper=self._config.zookeeper_server(), delete_hndlr=self._del_uves, logger=self._logger, cluster_id=self._config.cluster_id()) def _make_if_cdata(self, data): if_cdata = {} t = time.time() for dev in data: if 'snmp' in data[dev]: if 'ifMib' in data[dev]['snmp']: if 'ifTable' in data[dev]['snmp']['ifMib']: if_cdata[dev] = dict(map(lambda x: ( x['ifIndex'], (x['ifOperStatus'], t)), filter(lambda x: 'ifOperStatus' in x\ and 'ifDescr' in x, data[dev][ 'snmp']['ifMib']['ifTable']))) elif 'ifOperStatus' in data[dev]['snmp']: if_cdata[dev] = dict((k, (v, t)) for k, v in data[dev]['snmp']['ifOperStatus'].items()) return if_cdata def _delete_if_data(self, dev): if dev in self._if_data: del self._if_data[dev] def _set_status(self, _dict, dev, intf, val): if dev not in _dict: _dict[dev] = {} _dict[dev][intf] = val def _check_and_update_ttl(self, up2down): t = time.time() expry = 3 * self._fast_scan_freq for dev in self._if_data: for intf in self._if_data[dev]: if self._if_data[dev][intf][0] == 1: if t - self._if_data[dev][intf][1] > expry: self._set_status(up2down, dev, intf, 7) #no resp self._if_data[dev][intf] = (7, t) def _get_if_changes(self, if_cdata): down2up, up2down, others = {}, {}, {} for dev in if_cdata: if dev in self._if_data: for intf in if_cdata[dev]: if intf in self._if_data[dev]: if if_cdata[dev][intf][0] != self._if_data[dev][ intf][0]: if self._if_data[dev][intf][0] == 1: self._set_status(up2down, dev, intf, if_cdata[dev][intf][0]) elif if_cdata[dev][intf][0] == 1: self._set_status(down2up, dev, intf, if_cdata[dev][intf][0]) else: self._set_status(others, dev, intf, if_cdata[dev][intf][0]) self._if_data[dev][intf] = if_cdata[dev][intf] else: self._if_data[dev] = if_cdata[dev] for intf in self._if_data[dev]: if self._if_data[dev][intf][0] == 1: self._set_status(down2up, dev, intf, if_cdata[dev][intf][0]) else: self._set_status(others, dev, intf, if_cdata[dev][intf][0]) return down2up, up2down, others def _chk_if_change(self, data): if_cdata = self._make_if_cdata(data) down2up, up2down, others = self._get_if_changes(if_cdata) self._check_and_update_ttl(up2down) self._logger.debug('@chk_if_change: down2up(%s), up2down(%s), ' \ 'others(%s)' % (', '.join(down2up.keys()), ', '.join(up2down.keys()), ', '.join(others.keys()))) return down2up, up2down, others def _extra_call_params(self): if self._state != 'full_scan': return dict(restrict='ifOperStatus') return {} def _analyze(self, data): ret = True time = self._fast_scan_freq if self._state != 'full_scan': down2up, up2down, others = self._chk_if_change(data) if down2up: self._state = 'full_scan' time = self._mnt.add() elif self._mnt.ready4full_scan(): self._state = 'full_scan' time = 0 elif up2down: self.uve.send_ifstatus_update(self._if_data) ret = False sret = 'chngd: ' + self._state + ', time: ' + str(time) else: self._state = 'fast_scan' self._if_data = self._make_if_cdata(data) self.uve.send_ifstatus_update(self._if_data) sret = 'chngd: %d' % len(self._if_data) self._logger.debug('@do_work(analyze):State %s(%d)->%s!' % ( self._state, len(data), str(sret))) return ret, time def notify(self, svc, msg='', up=True, servers=''): self.uve.conn_state_notify(svc, msg, up, servers) def sleep_time(self, newtime=None): if newtime: self._sleep_time = newtime else: self._sleep_time = self._config.frequency() self._fast_scan_freq = self._config.fast_scan_freq() if self._fast_scan_freq > self._sleep_time / 2: self._fast_scan_freq = self._sleep_time / 2 return self._sleep_time def _setup_io(self): cdir = mkdtemp() input_file = os.path.join(cdir, 'in.data') output_file = os.path.join(cdir, 'out.data') return cdir, input_file, output_file def _create_input(self, input_file, output_file, devices, i, restrict=None): if isinstance(devices[0], DeviceDict): devices = DeviceConfig.populate_cfg(devices) with open(input_file, 'wb') as f: data = dict(out=output_file, netdev=devices, instance=i) if restrict: data['restrict'] = restrict pickle.dump(data, f) f.flush() def _run_scanner(self, input_file, output_file, i): proc = subprocess.Popen('contrail-snmp-scanner --input %s' % ( input_file), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) self._cleanup = (proc, output_file) o,e = proc.communicate() self._cleanup = None self._logger.debug('@run_scanner(%d): scan done with %d\nstdout:' \ '\n%s\nstderr:\n%s\n' % (i, proc.returncode, o, e)) with open(output_file, 'rb') as f: d = pickle.load(f) self._logger.debug('@run_scanner(%d): loaded %s' % (i, output_file)) return d def _cleanup_io(self, cdir, input_file, output_file): os.unlink(input_file) os.unlink(output_file) os.rmdir(cdir) def _send_uve(self, d): for dev, data in d.items(): if dev: self.uve.send(data['snmp']) self.uve.send_flow_uve({'name': dev, 'flow_export_source_ip': data['flow_export_source_ip']}) self.find_fix_name(data['name'], dev) self._logger.debug('@send_uve:Processed %d!' % (len(d))) def _send_snmp_collector_uve(self, members, partitions, prouters): snmp_collector_info = SnmpCollectorInfo() if self._members != members: self._members = members snmp_collector_info.members = members if self._partitions != partitions: self._partitions = partitions snmp_collector_info.partitions = partitions new_prouters = {p.name: p for p in prouters} if self._prouters.keys() != new_prouters.keys(): deleted_prouters = [v for p, v in self._prouters.iteritems() \ if p not in new_prouters] self._del_uves(deleted_prouters) self._prouters = new_prouters snmp_collector_info.prouters = self._prouters.keys() if snmp_collector_info != SnmpCollectorInfo(): snmp_collector_info.name = self._hostname SnmpCollectorUVE(data=snmp_collector_info).send() # end _send_snmp_collector_uve def _del_uves(self, l): with self._sem: for dev in l: self._delete_if_data(dev.name) self.uve.delete(dev) def do_work(self, i, devices): self._logger.debug('@do_work(%d):started (%d)...' % (i, len(devices))) sleep_time = self._fast_scan_freq if devices: with self._sem: self._work_set = devices cdir, input_file, output_file = self._setup_io() self._create_input(input_file, output_file, devices, i, **self._extra_call_params()) data = self._run_scanner(input_file, output_file, i) self._cleanup_io(cdir, input_file, output_file) do_send, sleep_time = self._analyze(data) if do_send: self._send_uve(data) gevent.sleep(0) del self._work_set self._logger.debug('@do_work(%d):Processed %d!' % (i, len(devices))) return sleep_time def find_fix_name(self, cfg_name, snmp_name): if snmp_name != cfg_name: self._logger.debug('@find_fix_name: snmp name %s differs from ' \ 'configured name %s, fixed for this run' % ( snmp_name, cfg_name)) for d in self._work_set: if d.name == cfg_name: d.name = snmp_name return def sighup_handler(self): if self._config._args.conf_file: config = ConfigParser.SafeConfigParser() config.read(self._config._args.conf_file) if 'DEFAULTS' in config.sections(): try: collectors = config.get('DEFAULTS', 'collectors') if type(collectors) is str: collectors = collectors.split() new_chksum = hashlib.md5("".join(collectors)).hexdigest() if new_chksum != self._chksum: self._chksum = new_chksum self._config.random_collectors = \ random.sample(collectors, len(collectors)) # Reconnect to achieve load-balance irrespective of list self.uve.sandesh_reconfig_collectors( self._config.random_collectors) except ConfigParser.NoOptionError as e: pass # end sighup_handler def _snmp_walker(self): i = 0 while True: self._logger.debug('@run: ittr(%d)' % i) devices = map(lambda e: DeviceDict(e[0].split(':')[-1], e[1].obj), self._config_handler.get_physical_routers()) if self._consistent_scheduler.schedule(devices): members = self._consistent_scheduler.members() partitions = self._consistent_scheduler.partitions() work_items = self._consistent_scheduler.work_items() self._send_snmp_collector_uve(members, partitions, work_items) sleep_time = self.do_work(i, work_items) self._logger.debug('done work %s' % str(self._prouters.keys())) i += 1 gevent.sleep(sleep_time) else: gevent.sleep(1) # end _snmp_walker def run(self): """ @sighup SIGHUP handler to indicate configuration changes """ gevent.signal(signal.SIGHUP, self.sighup_handler) self.gevs = [ gevent.spawn(self._config_handler.start), gevent.spawn(self._snmp_walker) ] try: gevent.joinall(self.gevs) except KeyboardInterrupt: self._logger.error('Exiting on ^C') except gevent.GreenletExit: self._logger.error('Exiting on gevent-kill') finally: self._logger.error('stopping everything!') self.stop() # end run def stop(self): self.uve.killall() l = len(self.gevs) for i in range(0, l): self._logger.error('killing %d of %d' % (i+1, l)) self.gevs[0].kill() self._logger.error('joining %d of %d' % (i+1, l)) self.gevs[0].join() self._logger.error('stopped %d of %d' % (i+1, l)) self.gevs.pop(0) self._consistent_scheduler.finish()
class Controller(object): def __init__(self, config): self._config = config self._hostname = socket.gethostname() self.analytic_api = AnalyticApiClient(self._config) self._config.random_collectors = self._config.collectors() self._chksum = "" self._api_server_checksum = "" if self._config.collectors(): self._chksum = hashlib.md5("".join(self._config.collectors())).hexdigest() self._config.random_collectors = random.sample(self._config.collectors(), \ len(self._config.collectors())) if self._config.api_server_list(): self._api_server_checksum = hashlib.md5("".join( self._config.api_server_list())).hexdigest() random_api_servers = random.sample( self._config.api_server_list(), len(self._config.api_server_list())) self._config.set_api_server_list(random_api_servers) self.uve = LinkUve(self._config) self._logger = self.uve.logger() self.sleep_time() self._keep_running = True self._vnc = None self._members = None self._partitions = None self._prouters = {} self._vrouter_l2ifs = {} self._old_vrouter_l2ifs = {} def stop(self): self._keep_running = False def sleep_time(self, newtime=None): if newtime: self._sleep_time = newtime else: self._sleep_time = self._config.frequency() return self._sleep_time def get_vrouters(self): self.analytic_api.get_vrouters(True) self.vrouters = {} self.vrouter_ips = {} self.vrouter_macs = {} for vr in self.analytic_api.list_vrouters(): cfilt = ['VrouterAgent:phy_if', 'VrouterAgent:self_ip_list', 'VRouterL2IfInfo'] try: d = self.analytic_api.get_vrouter(vr, ','.join(cfilt)) except Exception as e: traceback.print_exc() print str(e) d = {} if 'VrouterAgent' not in d or\ 'self_ip_list' not in d['VrouterAgent'] or\ 'phy_if' not in d['VrouterAgent']: continue self.vrouters[vr] = {'ips': d['VrouterAgent']['self_ip_list'], 'if': d['VrouterAgent']['phy_if'] } try: self.vrouters[vr]['l2_if'] = d['VRouterL2IfInfo']['if_info'] except KeyError: pass for ip in d['VrouterAgent']['self_ip_list']: self.vrouter_ips[ip] = vr # index for intf in d['VrouterAgent']['phy_if']: try: self.vrouter_macs[intf['mac_address']] = {} self.vrouter_macs[intf['mac_address']]['vrname'] = vr self.vrouter_macs[intf['mac_address']]['ifname'] = intf['name'] except: continue def get_prouters(self): self.analytic_api.get_prouters(True) self.prouters = [] for pr in self.analytic_api.list_prouters(): try: data = self.analytic_api.get_prouter(pr, 'PRouterEntry') if data: self.prouters.append(PRouter(pr, data)) except Exception as e: traceback.print_exc() print str(e) def _is_linkup(self, prouter, ifindex): if 'PRouterEntry' in prouter.data and \ 'ifIndexOperStatusTable' in prouter.data['PRouterEntry']: status = filter(lambda x: x['ifIndex'] == ifindex, prouter.data['PRouterEntry']['ifIndexOperStatusTable']) if status and status[0]['ifOperStatus'] == 1: return True return False def _add_link(self, prouter, remote_system_name, local_interface_name, remote_interface_name, local_interface_index, remote_interface_index, link_type): # If the remote_system_name or remote_interface_name is None, do not # add this link in the link_table. if not all([remote_system_name, remote_interface_name]): return False d = dict(remote_system_name=remote_system_name, local_interface_name=local_interface_name, remote_interface_name=remote_interface_name, local_interface_index=local_interface_index, remote_interface_index=remote_interface_index, type=link_type) if link_type == RemoteType.VRouter: l2_if = self.vrouters[remote_system_name].get('l2_if') if l2_if and remote_interface_name in l2_if: if l2_if[remote_interface_name]['remote_system_name'] != \ prouter.name: return False if self._is_linkup(prouter, local_interface_index): if prouter.name in self.link: self.link[prouter.name].append(d) else: self.link[prouter.name] = [d] return True return False def _chk_lnk(self, pre, index): if 'ifIndexOperStatusTable' in pre: for d in pre['ifIndexOperStatusTable']: if d['ifIndex'] == index: return d['ifOperStatus'] == 1 return False def _send_topology_uve(self, members, partitions, prouters): topology_info = TopologyInfo() if self._members != members: self._members = members topology_info.members = members if self._partitions != partitions: self._partitions = partitions topology_info.partitions = partitions new_prouters = {p.name: p for p in prouters} if self._prouters.keys() != new_prouters.keys(): deleted_prouters = [v for p, v in self._prouters.iteritems() \ if p not in new_prouters] self._del_uves(deleted_prouters) self._prouters = new_prouters topology_info.prouters = self._prouters.keys() if topology_info != TopologyInfo(): topology_info.name = self._hostname TopologyUVE(data=topology_info).send() # end _send_topology_uve def bms_links(self, prouter, ifm): if self._vnc: try: for li in self._vnc.logical_interfaces_list()[ 'logical-interfaces']: if prouter.name in li['fq_name']: lif = self._vnc.logical_interface_read(id=li['uuid']) for vmif in lif.get_virtual_machine_interface_refs(): vmi = self._vnc.virtual_machine_interface_read( id=vmif['uuid']) for mc in vmi.virtual_machine_interface_mac_addresses.get_mac_address(): ifi = [k for k in ifm if ifm[k] in li[ 'fq_name']][0] rsys = '-'.join(['bms', 'host'] + mc.split( ':')) if self._add_link( prouter=prouter, remote_system_name=rsys, local_interface_name=li['fq_name'][ -1], remote_interface_name='em0',#no idea local_interface_index=ifi, remote_interface_index=1, #dont know TODO:FIX link_type=RemoteType.BMS): pass except: traceback.print_exc() self._vnc = None # refresh def compute(self): self.link = {} self._old_vrouter_l2ifs = self._vrouter_l2ifs self._vrouter_l2ifs = {} for prouter in self.constnt_schdlr.work_items(): pr, d = prouter.name, prouter.data if 'PRouterEntry' not in d or 'ifTable' not in d['PRouterEntry']: continue self.link[pr] = [] lldp_ints = [] ifm = dict(map(lambda x: (x['ifIndex'], x['ifDescr']), d['PRouterEntry']['ifTable'])) self.bms_links(prouter, ifm) for pl in d['PRouterEntry']['lldpTable']['lldpRemoteSystemsData']: if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Cisco'): loc_pname = [x for x in d['PRouterEntry']['lldpTable'][ 'lldpLocalSystemData']['lldpLocPortTable'] if x[ 'lldpLocPortNum'] == pl['lldpRemLocalPortNum']][ 0]['lldpLocPortDesc'] pl['lldpRemLocalPortNum'] = [k for k in ifm if ifm[ k] == loc_pname][0] elif d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Arista'): loc_pname = [x for x in d['PRouterEntry']['lldpTable'][ 'lldpLocalSystemData']['lldpLocPortTable'] if x[ 'lldpLocPortNum'] == pl['lldpRemLocalPortNum']][ 0]['lldpLocPortId'] pl['lldpRemLocalPortNum'] = [k for k in ifm if ifm[ k] == loc_pname][0] if pl['lldpRemLocalPortNum'] in ifm and self._chk_lnk( d['PRouterEntry'], pl['lldpRemLocalPortNum']): if pl['lldpRemPortId'].isdigit(): rii = int(pl['lldpRemPortId']) else: try: if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Arista'): rpn = filter(lambda y: y['lldpLocPortId'] == pl[ 'lldpRemPortId'], [ x for x in self.prouters if x.name == pl[ 'lldpRemSysName']][0].data['PRouterEntry'][ 'lldpTable']['lldpLocalSystemData'][ 'lldpLocPortTable'])[0]['lldpLocPortId'] else: rpn = filter(lambda y: y['lldpLocPortId'] == pl[ 'lldpRemPortId'], [ x for x in self.prouters if x.name == pl[ 'lldpRemSysName']][0].data['PRouterEntry'][ 'lldpTable']['lldpLocalSystemData'][ 'lldpLocPortTable'])[0]['lldpLocPortDesc'] rii = filter(lambda y: y['ifDescr'] == rpn, [ x for x in self.prouters \ if x.name == pl['lldpRemSysName']][0].data[ 'PRouterEntry']['ifTable'])[0]['ifIndex'] except: rii = 0 if d['PRouterEntry']['lldpTable']['lldpLocalSystemData'][ 'lldpLocSysDesc'].startswith('Arista'): if self._add_link( prouter=prouter, remote_system_name=pl['lldpRemSysName'], local_interface_name=ifm[pl['lldpRemLocalPortNum']], remote_interface_name=pl['lldpRemPortId'], local_interface_index=pl['lldpRemLocalPortNum'], remote_interface_index=rii, link_type=RemoteType.PRouter): lldp_ints.append(ifm[pl['lldpRemLocalPortNum']]) else: if self._add_link( prouter=prouter, remote_system_name=pl['lldpRemSysName'], local_interface_name=ifm[pl['lldpRemLocalPortNum']], remote_interface_name=pl['lldpRemPortDesc'], local_interface_index=pl['lldpRemLocalPortNum'], remote_interface_index=rii, link_type=RemoteType.PRouter): lldp_ints.append(ifm[pl['lldpRemLocalPortNum']]) vrouter_l2ifs = {} if 'fdbPortIfIndexTable' in d['PRouterEntry']: dot1d2snmp = map (lambda x: ( x['dot1dBasePortIfIndex'], x['snmpIfIndex']), d['PRouterEntry']['fdbPortIfIndexTable']) dot1d2snmp_dict = dict(dot1d2snmp) if 'fdbPortTable' in d['PRouterEntry']: for mac_entry in d['PRouterEntry']['fdbPortTable']: if mac_entry['mac'] in self.vrouter_macs: vrouter_mac_entry = self.vrouter_macs[mac_entry['mac']] vr_name = vrouter_mac_entry['vrname'] vr_ifname = vrouter_mac_entry['ifname'] fdbport = mac_entry['dot1dBasePortIfIndex'] try: snmpport = dot1d2snmp_dict[fdbport] ifname = ifm[snmpport] except: continue is_lldp_int = any(ifname == lldp_int for lldp_int in lldp_ints) if is_lldp_int: continue if self._add_link( prouter=prouter, remote_system_name=vr_name, local_interface_name=ifname, remote_interface_name=vr_ifname, local_interface_index=snmpport, remote_interface_index=1, #dont know TODO:FIX link_type=RemoteType.VRouter): if vr_name not in vrouter_l2ifs: vrouter_l2ifs[vr_name] = {} vrouter_l2ifs[vr_name][vr_ifname] = { 'remote_system_name': prouter.name, 'remote_if_name': ifname, } for arp in d['PRouterEntry']['arpTable']: if arp['ip'] in self.vrouter_ips: if arp['mac'] in map(lambda x: x['mac_address'], self.vrouters[self.vrouter_ips[arp['ip']]]['if']): vr_name = self.vrouter_macs[arp['mac']]['vrname'] vr_ifname = self.vrouter_macs[arp['mac']]['ifname'] try: if vrouter_l2ifs[vr_name][vr_ifname]\ ['remote_system_name'] == prouter.name: del vrouter_l2ifs[vr_name][vr_ifname] if not vrouter_l2ifs[vr_name]: del vrouter_l2ifs[vr_name] continue except KeyError: pass if ifm[arp['localIfIndex']].startswith('vlan'): continue if ifm[arp['localIfIndex']].startswith('irb'): continue is_lldp_int = any(ifm[arp['localIfIndex']] == lldp_int for lldp_int in lldp_ints) if is_lldp_int: continue if self._add_link( prouter=prouter, remote_system_name=vr_name, local_interface_name=ifm[arp['localIfIndex']], remote_interface_name=vr_ifname, local_interface_index=arp['localIfIndex'], remote_interface_index=1, #dont know TODO:FIX link_type=RemoteType.VRouter): pass for vr, intf in vrouter_l2ifs.iteritems(): if vr in self._vrouter_l2ifs: self._vrouter_l2ifs[vr].update(vrouter_l2ifs[vr]) else: self._vrouter_l2ifs[vr] = intf def send_uve(self): old_vrs = set(self._old_vrouter_l2ifs.keys()) new_vrs = set(self._vrouter_l2ifs.keys()) del_vrs = old_vrs - new_vrs add_vrs = new_vrs - old_vrs same_vrs = old_vrs.intersection(new_vrs) for vr in del_vrs: vr_l2info = VRouterL2IfInfo(name=vr, deleted=True) VRouterL2IfUVE(data=vr_l2info).send() for vr in add_vrs: if_info = {} for vrif, remif_info in self._vrouter_l2ifs[vr].iteritems(): if_info[vrif] = RemoteIfInfo(remif_info['remote_system_name'], remif_info['remote_if_name']) vr_l2info = VRouterL2IfInfo(name=vr, if_info=if_info) VRouterL2IfUVE(data=vr_l2info).send() for vr in same_vrs: if self._vrouter_l2ifs[vr] != self._old_vrouter_l2ifs[vr]: if_info = {} for vrif, remif_info in self._vrouter_l2ifs[vr].iteritems(): if_info[vrif] = RemoteIfInfo( remif_info['remote_system_name'], remif_info['remote_if_name']) vr_l2info = VRouterL2IfInfo(name=vr, if_info=if_info) VRouterL2IfUVE(data=vr_l2info).send() self.uve.send(self.link) def switcher(self): gevent.sleep(0) def scan_data(self): t = [] t.append(gevent.spawn(self.get_vrouters)) t.append(gevent.spawn(self.get_prouters)) gevent.joinall(t) def _del_uves(self, prouters): with self._sem: for prouter in prouters: self.uve.delete(prouter.name) def sighup_handler(self): if self._config._args.conf_file: config = ConfigParser.SafeConfigParser() config.read(self._config._args.conf_file) if 'DEFAULTS' in config.sections(): try: collectors = config.get('DEFAULTS', 'collectors') if type(collectors) is str: collectors = collectors.split() new_chksum = hashlib.md5("".join(collectors)).hexdigest() if new_chksum != self._chksum: self._chksum = new_chksum self._config.random_collectors = \ random.sample(collectors, len(collectors)) # Reconnect to achieve load-balance irrespective of list self.uve.sandesh_reconfig_collectors( self._config.random_collectors) except ConfigParser.NoOptionError as e: pass if 'API_SERVER' in config.sections(): try: api_servers = config.get('API_SERVER', 'api_server_list') except ConfigParser.NoOptionError: pass else: if isinstance(api_servers, basestring): api_servers = api_servers.split() new_api_server_checksum = hashlib.md5("".join( api_servers)).hexdigest() if new_api_server_checksum != self._api_server_checksum: self._api_server_checksum = new_api_server_checksum random_api_servers = random.sample(api_servers, len(api_servers)) self._config.set_api_server_list(random_api_servers) self._vnc = None # end sighup_handler def run(self): """ @sighup SIGHUP handler to indicate configuration changes """ gevent.signal(signal.SIGHUP, self.sighup_handler) self._sem = Semaphore() self.constnt_schdlr = ConsistentScheduler( self.uve._moduleid, zookeeper=self._config.zookeeper_server(), delete_hndlr=self._del_uves, logger=self._logger, cluster_id=self._config.cluster_id()) while self._keep_running: if not self._vnc: self._vnc = self._config.vnc_api() self.scan_data() if self.constnt_schdlr.schedule(self.prouters): members = self.constnt_schdlr.members() partitions = self.constnt_schdlr.partitions() self._send_topology_uve(members, partitions, self.constnt_schdlr.work_items()) try: with self._sem: self.compute() self.send_uve() except Exception as e: traceback.print_exc() print str(e) gevent.sleep(self._sleep_time) else: gevent.sleep(1) self.constnt_schdlr.finish()