def test_format_endpoints(): input_data = list([{'attachment-point': {'switch-interface': {'interface': 'ethernet16', 'switch': 'leaf02'}, 'type': 'switch-interface'}, 'attachment-point-state': 'learned', 'created-since': '2017-09-18T16:28:34.694Z', 'detail': 'true', 'interface': 'ethernet16', 'ip-address': [{'ip-address': '10.0.0.101', 'ip-state': 'learned', 'mac': 'f8:b1:56:fe:f2:de', 'segment': 'prod', 'tenant': 'FLOORPLATE'}], 'leaf-group': '00:00:f4:8e:38:16:a3:73', 'mac': 'f8:b1:56:fe:f2:de', 'nat-endpoint': False, 'remote': False, 'segment': 'prod', 'state': 'Active', 'switch': 'leaf02', 'tenant': 'FLOORPLATE', 'vlan': -1}, {'attachment-point': {'switch-interface': {'interface': 'ethernet42', 'switch': 'leaf01'}, 'type': 'switch-interface'}, 'attachment-point-state': 'learned', 'created-since': '2017-07-11T23:56:23.888Z', 'detail': 'true', 'interface': 'ethernet42', 'leaf-group': '00:00:f4:8e:38:16:b3:73', 'mac': '20:4c:9e:5f:e3:a3', 'nat-endpoint': False, 'remote': False, 'segment': 'to-core-router', 'state': 'Active', 'switch': 'leaf01', 'tenant': 'EXTERNAL', 'vlan': -1}]) output = BcfProxy.format_endpoints(input_data) answer = list([{'ip-address': '10.0.0.101', 'mac': 'f8:b1:56:fe:f2:de', 'segment': 'prod', 'tenant': 'FLOORPLATE', 'name': None}]) assert str(answer) == str(output)
class Update_Switch_State(Monitor_Helper_Base): ''' handle periodic process, determine if switch state updated ''' def __init__(self): super(Update_Switch_State, self).__init__() self.logger = module_logger.logger self.mod_name = self.__class__.__name__ self.retval = {} self.times = 0 self.owner = None self.controller = {} self.controller['URI'] = None self.controller['USER'] = None self.controller['PASS'] = None self.bcf = None self.first_time = True self.endpoint_states = defaultdict(dict) self.m_queue = Queue.Queue() def return_endpoint_state(self): return self.endpoint_states def first_run(self): ''' do some pre-run setup/configuration ''' if self.configured: self.controller['URI'] = str( self.mod_configuration['controller_uri']) self.controller['USER'] = str( self.mod_configuration['controller_user']) self.controller['PASS'] = str( self.mod_configuration['controller_pass']) myauth = {} myauth['password'] = self.controller['PASS'] myauth['user'] = self.controller['USER'] try: self.bcf = BcfProxy(self.controller['URI'], auth=myauth) except BaseException: self.logger.error('BcfProxy coult not connect to {0}'.format( self.controller['URI'])) else: pass @staticmethod def make_hash(item): ''' hash the metadata in a sane way''' h = hashlib.new('ripemd160') pre_h = str() post_h = None # nodhcp -> dhcp withname makes different hashes # {u'tenant': u'FLOORPLATE', u'mac': u'ac:87:a3:2b:7f:12', u'segment': u'prod', u'name': None, u'ip-address': u'10.179.0.100'}}^ # {u'tenant': u'FLOORPLATE', u'mac': u'ac:87:a3:2b:7f:12', u'segment': u'prod', u'name': u'demo-laptop', u'ip-address': u'10.179.0.100'}} # ^^^ make different hashes if name is included # for word in ['tenant', 'mac', 'segment', 'name', 'ip-address']: for word in ['tenant', 'mac', 'segment', 'ip-address']: pre_h = pre_h + str(item.get(str(word), 'missing')) h.update(pre_h) post_h = h.hexdigest() return post_h def get_endpoint_state(self, my_hash): if my_hash in self.endpoint_states: return self.endpoint_states[my_hash]['state'] return None def get_endpoint_ip(self, my_hash): if my_hash in self.endpoint_states: return self.endpoint_states[my_hash]['endpoint']['ip-address'] return None def shutdown_endpoint(self, my_hash, next_state='SHUTDOWN'): if my_hash in self.endpoint_states: if self.get_endpoint_state(my_hash) != next_state: my_ip = self.get_endpoint_ip(my_hash) self.bcf.shutdown_ip(my_ip) self.change_endpoint_state(my_hash) self.logger.debug('endpoint:{0}:{1}:{2}'.format( my_hash, my_ip, next_state)) return True return False def mirror_endpoint(self, my_hash, next_state='MIRRORING'): if my_hash in self.endpoint_states: if self.get_endpoint_state(my_hash) != next_state: my_ip = self.get_endpoint_ip(my_hash) self.bcf.mirror_ip(my_ip) self.change_endpoint_state(my_hash) self.logger.debug('endpoint:{0}:{1}:{2}'.format( my_hash, my_ip, next_state)) return True return False def make_known_endpoint(self, my_hash, next_state='KNOWN'): if my_hash in self.endpoint_states: if self.get_endpoint_state(my_hash) != next_state: my_ip = self.get_endpoint_ip(my_hash) self.bcf.unmirror_ip(my_ip) self.change_endpoint_state(my_hash, 'KNOWN') self.logger.debug('endpoint:{0}:{1}:{2}'.format( my_hash, my_ip, next_state)) return True return False def make_endpoint_dict(self, my_hash, state, data): self.endpoint_states[my_hash]['state'] = state self.endpoint_states[my_hash]['next-state'] = 'NONE' self.endpoint_states[my_hash]['endpoint'] = data def change_endpoint_state(self, my_hash, new_state=None): self.endpoint_states[my_hash][ 'state'] = new_state or self.endpoint_states[my_hash]['next-state'] self.endpoint_states[my_hash]['next-state'] = 'NONE' def find_new_machines(self, machines): '''parse switch structure to find new machines added to network since last call''' if self.first_time: self.first_time = False # TODO db call to see if really need to run things for machine in machines: h = self.make_hash(machine) self.logger.critical( 'adding address to known systems {0}'.format(machine)) self.make_endpoint_dict(h, 'KNOWN', machine) else: for machine in machines: h = self.make_hash(machine) if h not in self.endpoint_states: self.logger.critical( '***** detected new address {0}'.format(machine)) self.make_endpoint_dict(h, 'UNKNOWN', machine) def print_endpoint_state(self): def same_old(logger, state, letter, endpoint_states): logger.debug('*******{0}*********'.format(state)) out_flag = False for my_hash in endpoint_states.keys(): my_dict = endpoint_states[my_hash] if my_dict['state'] == state: out_flag = True logger.debug('{0}:{1}:{2}->{3}:{4}'.format( letter, my_hash, my_dict['state'], my_dict['next-state'], my_dict['endpoint'])) if not out_flag: logger.debug('None') states = [('K', 'KNOWN'), ('U', 'UNKNOWN'), ('M', 'MIRRORING'), ('S', 'SHUTDOWN'), ('R', 'REINVESTIGATING')] for l, s in states: same_old(self.logger, s, l, self.endpoint_states) self.logger.debug('****************') def update_endpoint_state(self): '''Handles Get requests''' self.retval['service'] = self.owner.mod_name + ':' + self.mod_name self.retval['times'] = self.times self.retval['machines'] = None self.retval['resp'] = 'bad' current = None parsed = None machines = {} try: current = self.bcf.get_endpoints() parsed = self.bcf.format_endpoints(current) machines = parsed except BaseException: self.logger.error('Could not establish connection to {0}.'.format( self.controller['URI'])) self.retval[ 'controller'] = 'Could not establish connection to {0}.'.format( self.controller['URI']) self.logger.debug('MACHINES:{0}'.format(machines)) self.find_new_machines(machines) self.print_endpoint_state() self.retval['machines'] = parsed self.retval['resp'] = 'ok' self.times = self.times + 1 return json.dumps(self.retval)
class Handle_Periodic(Monitor_Helper_Base): ''' handle periodic process, determine if switch state updated ''' def __init__(self): super(Handle_Periodic, self).__init__() self.logger = module_logger self.mod_name = self.__class__.__name__ self.retval = {} self.times = 0 self.owner = None self.controller = {} self.controller['URI'] = None self.controller['USER'] = None self.controller['PASS'] = None self.bcf = None self.first_time = True self.prev_endpoints = {} self.new_endpoints = {} self.mirroring = {} self.shutdown = {} self.do_rabbit = True self.m_queue = Queue.Queue() self.rabbit_connection_local = None self.rabbit_channel_local = None if getenv('SKIPRABBIT') is None: module_logger.critical('handle_periodic skipping rabbit') self.do_rabbit = False else: module_logger.critical('handle_periodic starting rabbit') self.start_rabbit() # TODO init the rabbitmq # rabbit def start_rabbit(self): ''' start the rabbit negotiations ''' self.init_rabbit() if self.do_rabbit: self.start_channel(self.rabbit_channel_local, callback, 'poseidon_NBCA') def make_rabbit_connection(self, host, exchange, queue_name, keys): # pragma: no cover ''' Continuously loops trying to connect to rabbitmq, once connected declares the exchange and queue for processing algorithm results. ''' wait = True channel = None connection = None total_sleep = 30 while wait and total_sleep > 0: try: connection = pika.BlockingConnection( pika.ConnectionParameters(host=host)) channel = connection.channel() channel.exchange_declare(exchange=exchange, type='topic') channel.queue_declare(queue=queue_name, exclusive=True) self.logger.debug( '************* connected to {0} rabbitMQ'.format(host)) wait = False except Exception as e: self.logger.debug( '************* waiting for {0} rabbitQM'.format(host)) self.logger.debug(str(e)) time.sleep(2) total_sleep -= 2 wait = True if wait: self.do_rabbit = False if isinstance(keys, types.ListType) and not wait: for key in keys: self.logger.debug( 'array adding key:{0} to rabbitmq channel'.format(key)) channel.queue_bind(exchange=exchange, queue=queue_name, routing_key=key) if isinstance(keys, types.StringType) and not wait: self.logger.debug( 'string adding key:{0} to rabbitmq channel'.format(keys)) channel.queue_bind(exchange=exchange, queue=queue_name, routing_key=keys) return channel, connection def init_rabbit(self): # pragma: no cover ''' init_rabbit ''' host = 'poseidon-rabbit' exchange = 'topic-poseidon-internal' queue_name = 'poseidon_NBCA' # binding_key = ['poseidon.algos.#', 'poseidon.action.#'] binding_key = ['poseidon.action.#'] retval = self.make_rabbit_connection( host, exchange, queue_name, binding_key) self.rabbit_channel_local = retval[0] self.rabbit_connection_local = retval[1] def start_channel(self, channel, mycallback, queue): ''' handle threading for a messagetype ''' self.logger.debug('about to start channel {0}'.format(channel)) channel.basic_consume( partial(mycallback, q=self.m_queue), queue=queue, no_ack=True) mq_recv_thread = threading.Thread(target=channel.start_consuming) mq_recv_thread.start() def first_run(self): ''' do some pre-run setup/configuration ''' if self.configured: self.controller['URI'] = str( self.mod_configuration['controller_uri']) self.controller['USER'] = str( self.mod_configuration['controller_user']) self.controller['PASS'] = str( self.mod_configuration['controller_pass']) myauth = {} myauth['password'] = self.controller['PASS'] myauth['user'] = self.controller['USER'] try: self.bcf = BcfProxy(self.controller['URI'], auth=myauth) except: self.logger.error( 'BcfProxy coult not connect to {0}'.format(self.controller['URI'])) else: pass @staticmethod def make_hash(item): ''' hash the metadata in a sane way''' h = hashlib.new('ripemd160') pre_h = str() post_h = None for word in ['tenant', 'mac', 'segment', 'name', 'ip-address']: pre_h = pre_h + str(item.get(str(word), 'missing')) h.update(pre_h) post_h = h.hexdigest() return post_h def handle_item(self, item): ''' perform an action based on rabbit item''' self.logger.debug('handle_item: {0}:{1}'.format(item, type(item))) itype = item[0] ivalue = item[1] ivalue = json.loads(ivalue) self.logger.debug( 'handle_item: ivalue json: {0}:{1}'.format(ivalue, type(ivalue))) if itype == 'poseidon.action.start_monitor': for my_hash, my_dict in ivalue.iteritems(): if my_hash in self.new_endpoints: v = self.new_endpoints.pop(my_hash) self.logger.debug( 'removed {0} from new_endpoints'.format(v)) else: self.logger.debug('could not find {0} in {1}'.format( my_hash, self.new_endpoints)) self.logger.debug( 'mirroring :{0}'.format(my_dict['ip-address'])) self.logger.debug( 'mirroring[{0}]={1}'.format(my_hash, my_dict)) self.bcf.mirror_ip(my_dict['ip-address']) self.mirroring[my_hash] = my_dict if itype == 'poseidon.action.endpoint_shutdown': self.logger.debug( 'endpoint_shutdown:{0}:{1}'.format(ivalue, type(ivalue))) for my_hash, my_dict in ivalue.iteritems(): bad_ip = my_dict.get('ip-address') if bad_ip is not None: self.logger.debug( '****** shutdown {0}:{1}'.format(bad_ip, ivalue)) self.bcf.shutdown_ip(bad_ip) self.shutdown[my_hash] = my_dict if itype == 'poseidon.action.stop_monitor': self.logger.debug('stop_monitor:{0}:{1}'.format(itype, ivalue)) for my_hash, my_dict in ivalue.iteritems(): self.logger.debug('stop_monitor_dict:{0}'.format(my_dict)) my_ip = my_dict.get('ip-address') if my_ip is not None: self.logger.debug('***** shutting down {0}'.format(my_ip)) self.bcf.unmirror_ip(my_ip) if my_hash in self.mirroring: self.mirroring.pop(my_hash) def get_rabbit_work(self): '''get work item from queue if exists''' # type , value workfound = False item = None self.logger.debug('about to look for work') try: item = self.m_queue.get(False) self.logger.debug('item:{0}'.format(item)) self.logger.debug('work found') workfound = True except Queue.Empty: pass self.logger.debug('done looking for work!') if workfound: self.handle_item(item) return item def find_new_machines(self, machines): '''parse switch structure to find new machines added to network since last call''' if self.first_time: self.first_time = False # TODO db call to see if really need to run things for machine in machines: h = self.make_hash(machine) module_logger.critical( 'adding address to known systems {0}'.format(machine)) self.prev_endpoints[h] = machine else: for machine in machines: h = self.make_hash(machine) if h not in self.prev_endpoints and h not in self.mirroring: module_logger.critical( '***** detected new address {0}'.format(machine)) self.new_endpoints[h] = machine def print_state(self): self.logger.debug('**************PREV*****************') for my_hash, my_dict in self.prev_endpoints.iteritems(): self.logger.debug('P:{0}:{1}'.format(my_hash, my_dict)) self.logger.debug('**************NEW******************') for my_hash, my_dict in self.new_endpoints.iteritems(): self.logger.debug('N:{0}:{1}'.format(my_hash, my_dict)) self.logger.debug('***********MIRRORING***************') for my_hash, my_dict in self.mirroring.iteritems(): self.logger.debug('M:{0}:{1}'.format(my_hash, my_dict)) self.logger.debug('***********SHUTDOWN****************') for my_hash, my_dict in self.shutdown.iteritems(): self.logger.debug('M:{0}:{1}'.format(my_hash, my_dict)) def send_new_machines(self): '''send listing of new machines to main for decisions''' for hashed, machine in self.new_endpoints.iteritems(): # TODO write findings to main r_exchange = 'topic-poseidon-internal' r_key = 'poseidon.action.new_machine' r_msg = json.dumps({hashed: machine}) self.rabbit_channel_local.basic_publish(exchange=r_exchange, routing_key=r_key, body=r_msg) def on_get(self, req, resp): '''Handles Get requests''' self.retval['service'] = self.owner.mod_name + ':' + self.mod_name self.retval['times'] = self.times self.retval['machines'] = None self.retval['resp'] = 'bad' current = None parsed = None machines = {} try: current = self.bcf.get_endpoints() parsed = self.bcf.format_endpoints(current) machines = parsed except: self.logger.error( 'Could not establish connection to {0}.'.format(self.controller['URI'])) self.retval['controller'] = 'Could not establish connection to {0}.'.format( self.controller['URI']) self.get_rabbit_work() self.logger.debug('MACHINES:{0}'.format(machines)) self.find_new_machines(machines) self.send_new_machines() self.print_state() self.retval['machines'] = parsed self.retval['resp'] = 'ok' # TODO change response to something reflecting success of traversal self.times = self.times + 1 resp.body = json.dumps(self.retval)