def test_format_endpoints():
input_data = list([{'attachment-point': {'switch-interface': {'interface': 'ethernet16',
'switch': 'leaf02'},
'type': 'switch-interface'},
'attachment-point-state': 'learned',
'created-since': '2017-09-18T16:28:34.694Z',
'detail': 'true',
'interface': 'ethernet16',
'ip-address': [{'ip-address': '10.0.0.101',
'ip-state': 'learned',
'mac': 'f8:b1:56:fe:f2:de',
'segment': 'prod',
'tenant': 'FLOORPLATE'}],
'leaf-group': '00:00:f4:8e:38:16:a3:73',
'mac': 'f8:b1:56:fe:f2:de',
'nat-endpoint': False,
'remote': False,
'segment': 'prod',
'state': 'Active',
'switch': 'leaf02',
'tenant': 'FLOORPLATE',
'vlan': -1},
{'attachment-point': {'switch-interface': {'interface': 'ethernet42',
'switch': 'leaf01'},
'type': 'switch-interface'},
'attachment-point-state': 'learned',
'created-since': '2017-07-11T23:56:23.888Z',
'detail': 'true',
'interface': 'ethernet42',
'leaf-group': '00:00:f4:8e:38:16:b3:73',
'mac': '20:4c:9e:5f:e3:a3',
'nat-endpoint': False,
'remote': False,
'segment': 'to-core-router',
'state': 'Active',
'switch': 'leaf01',
'tenant': 'EXTERNAL',
'vlan': -1}])
output = BcfProxy.format_endpoints(input_data)
answer = list([{'ip-address': '10.0.0.101', 'mac': 'f8:b1:56:fe:f2:de',
'segment': 'prod', 'tenant': 'FLOORPLATE', 'name': None}])
assert str(answer) == str(output)
class Update_Switch_State(Monitor_Helper_Base):
''' handle periodic process, determine if switch state updated '''
def __init__(self):
super(Update_Switch_State, self).__init__()
self.logger = module_logger.logger
self.mod_name = self.__class__.__name__
self.retval = {}
self.times = 0
self.owner = None
self.controller = {}
self.controller['URI'] = None
self.controller['USER'] = None
self.controller['PASS'] = None
self.bcf = None
self.first_time = True
self.endpoint_states = defaultdict(dict)
self.m_queue = Queue.Queue()
def return_endpoint_state(self):
return self.endpoint_states
def first_run(self):
''' do some pre-run setup/configuration '''
if self.configured:
self.controller['URI'] = str(
self.mod_configuration['controller_uri'])
self.controller['USER'] = str(
self.mod_configuration['controller_user'])
self.controller['PASS'] = str(
self.mod_configuration['controller_pass'])
myauth = {}
myauth['password'] = self.controller['PASS']
myauth['user'] = self.controller['USER']
try:
self.bcf = BcfProxy(self.controller['URI'], auth=myauth)
except BaseException:
self.logger.error('BcfProxy coult not connect to {0}'.format(
self.controller['URI']))
else:
pass
@staticmethod
def make_hash(item):
''' hash the metadata in a sane way'''
h = hashlib.new('ripemd160')
pre_h = str()
post_h = None
# nodhcp -> dhcp withname makes different hashes
# {u'tenant': u'FLOORPLATE', u'mac': u'ac:87:a3:2b:7f:12', u'segment': u'prod', u'name': None, u'ip-address': u'10.179.0.100'}}^
# {u'tenant': u'FLOORPLATE', u'mac': u'ac:87:a3:2b:7f:12', u'segment': u'prod', u'name': u'demo-laptop', u'ip-address': u'10.179.0.100'}}
# ^^^ make different hashes if name is included
# for word in ['tenant', 'mac', 'segment', 'name', 'ip-address']:
for word in ['tenant', 'mac', 'segment', 'ip-address']:
pre_h = pre_h + str(item.get(str(word), 'missing'))
h.update(pre_h)
post_h = h.hexdigest()
return post_h
def get_endpoint_state(self, my_hash):
if my_hash in self.endpoint_states:
return self.endpoint_states[my_hash]['state']
return None
def get_endpoint_ip(self, my_hash):
if my_hash in self.endpoint_states:
return self.endpoint_states[my_hash]['endpoint']['ip-address']
return None
def shutdown_endpoint(self, my_hash, next_state='SHUTDOWN'):
if my_hash in self.endpoint_states:
if self.get_endpoint_state(my_hash) != next_state:
my_ip = self.get_endpoint_ip(my_hash)
self.bcf.shutdown_ip(my_ip)
self.change_endpoint_state(my_hash)
self.logger.debug('endpoint:{0}:{1}:{2}'.format(
my_hash, my_ip, next_state))
return True
return False
def mirror_endpoint(self, my_hash, next_state='MIRRORING'):
if my_hash in self.endpoint_states:
if self.get_endpoint_state(my_hash) != next_state:
my_ip = self.get_endpoint_ip(my_hash)
self.bcf.mirror_ip(my_ip)
self.change_endpoint_state(my_hash)
self.logger.debug('endpoint:{0}:{1}:{2}'.format(
my_hash, my_ip, next_state))
return True
return False
def make_known_endpoint(self, my_hash, next_state='KNOWN'):
if my_hash in self.endpoint_states:
if self.get_endpoint_state(my_hash) != next_state:
my_ip = self.get_endpoint_ip(my_hash)
self.bcf.unmirror_ip(my_ip)
self.change_endpoint_state(my_hash, 'KNOWN')
self.logger.debug('endpoint:{0}:{1}:{2}'.format(
my_hash, my_ip, next_state))
return True
return False
def make_endpoint_dict(self, my_hash, state, data):
self.endpoint_states[my_hash]['state'] = state
self.endpoint_states[my_hash]['next-state'] = 'NONE'
self.endpoint_states[my_hash]['endpoint'] = data
def change_endpoint_state(self, my_hash, new_state=None):
self.endpoint_states[my_hash][
'state'] = new_state or self.endpoint_states[my_hash]['next-state']
self.endpoint_states[my_hash]['next-state'] = 'NONE'
def find_new_machines(self, machines):
'''parse switch structure to find new machines added to network
since last call'''
if self.first_time:
self.first_time = False
# TODO db call to see if really need to run things
for machine in machines:
h = self.make_hash(machine)
self.logger.critical(
'adding address to known systems {0}'.format(machine))
self.make_endpoint_dict(h, 'KNOWN', machine)
else:
for machine in machines:
h = self.make_hash(machine)
if h not in self.endpoint_states:
self.logger.critical(
'***** detected new address {0}'.format(machine))
self.make_endpoint_dict(h, 'UNKNOWN', machine)
def print_endpoint_state(self):
def same_old(logger, state, letter, endpoint_states):
logger.debug('*******{0}*********'.format(state))
out_flag = False
for my_hash in endpoint_states.keys():
my_dict = endpoint_states[my_hash]
if my_dict['state'] == state:
out_flag = True
logger.debug('{0}:{1}:{2}->{3}:{4}'.format(
letter, my_hash, my_dict['state'],
my_dict['next-state'], my_dict['endpoint']))
if not out_flag:
logger.debug('None')
states = [('K', 'KNOWN'), ('U', 'UNKNOWN'), ('M', 'MIRRORING'),
('S', 'SHUTDOWN'), ('R', 'REINVESTIGATING')]
for l, s in states:
same_old(self.logger, s, l, self.endpoint_states)
self.logger.debug('****************')
def update_endpoint_state(self):
'''Handles Get requests'''
self.retval['service'] = self.owner.mod_name + ':' + self.mod_name
self.retval['times'] = self.times
self.retval['machines'] = None
self.retval['resp'] = 'bad'
current = None
parsed = None
machines = {}
try:
current = self.bcf.get_endpoints()
parsed = self.bcf.format_endpoints(current)
machines = parsed
except BaseException:
self.logger.error('Could not establish connection to {0}.'.format(
self.controller['URI']))
self.retval[
'controller'] = 'Could not establish connection to {0}.'.format(
self.controller['URI'])
self.logger.debug('MACHINES:{0}'.format(machines))
self.find_new_machines(machines)
self.print_endpoint_state()
self.retval['machines'] = parsed
self.retval['resp'] = 'ok'
self.times = self.times + 1
return json.dumps(self.retval)
class Handle_Periodic(Monitor_Helper_Base):
''' handle periodic process, determine if switch state updated '''
def __init__(self):
super(Handle_Periodic, self).__init__()
self.logger = module_logger
self.mod_name = self.__class__.__name__
self.retval = {}
self.times = 0
self.owner = None
self.controller = {}
self.controller['URI'] = None
self.controller['USER'] = None
self.controller['PASS'] = None
self.bcf = None
self.first_time = True
self.prev_endpoints = {}
self.new_endpoints = {}
self.mirroring = {}
self.shutdown = {}
self.do_rabbit = True
self.m_queue = Queue.Queue()
self.rabbit_connection_local = None
self.rabbit_channel_local = None
if getenv('SKIPRABBIT') is None:
module_logger.critical('handle_periodic skipping rabbit')
self.do_rabbit = False
else:
module_logger.critical('handle_periodic starting rabbit')
self.start_rabbit()
# TODO init the rabbitmq
# rabbit
def start_rabbit(self):
''' start the rabbit negotiations '''
self.init_rabbit()
if self.do_rabbit:
self.start_channel(self.rabbit_channel_local,
callback, 'poseidon_NBCA')
def make_rabbit_connection(self, host, exchange, queue_name, keys): # pragma: no cover
'''
Continuously loops trying to connect to rabbitmq,
once connected declares the exchange and queue for
processing algorithm results.
'''
wait = True
channel = None
connection = None
total_sleep = 30
while wait and total_sleep > 0:
try:
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=host))
channel = connection.channel()
channel.exchange_declare(exchange=exchange, type='topic')
channel.queue_declare(queue=queue_name, exclusive=True)
self.logger.debug(
'************* connected to {0} rabbitMQ'.format(host))
wait = False
except Exception as e:
self.logger.debug(
'************* waiting for {0} rabbitQM'.format(host))
self.logger.debug(str(e))
time.sleep(2)
total_sleep -= 2
wait = True
if wait:
self.do_rabbit = False
if isinstance(keys, types.ListType) and not wait:
for key in keys:
self.logger.debug(
'array adding key:{0} to rabbitmq channel'.format(key))
channel.queue_bind(exchange=exchange,
queue=queue_name,
routing_key=key)
if isinstance(keys, types.StringType) and not wait:
self.logger.debug(
'string adding key:{0} to rabbitmq channel'.format(keys))
channel.queue_bind(exchange=exchange,
queue=queue_name, routing_key=keys)
return channel, connection
def init_rabbit(self): # pragma: no cover
''' init_rabbit '''
host = 'poseidon-rabbit'
exchange = 'topic-poseidon-internal'
queue_name = 'poseidon_NBCA'
# binding_key = ['poseidon.algos.#', 'poseidon.action.#']
binding_key = ['poseidon.action.#']
retval = self.make_rabbit_connection(
host, exchange, queue_name, binding_key)
self.rabbit_channel_local = retval[0]
self.rabbit_connection_local = retval[1]
def start_channel(self, channel, mycallback, queue):
''' handle threading for a messagetype '''
self.logger.debug('about to start channel {0}'.format(channel))
channel.basic_consume(
partial(mycallback, q=self.m_queue), queue=queue, no_ack=True)
mq_recv_thread = threading.Thread(target=channel.start_consuming)
mq_recv_thread.start()
def first_run(self):
''' do some pre-run setup/configuration '''
if self.configured:
self.controller['URI'] = str(
self.mod_configuration['controller_uri'])
self.controller['USER'] = str(
self.mod_configuration['controller_user'])
self.controller['PASS'] = str(
self.mod_configuration['controller_pass'])
myauth = {}
myauth['password'] = self.controller['PASS']
myauth['user'] = self.controller['USER']
try:
self.bcf = BcfProxy(self.controller['URI'], auth=myauth)
except:
self.logger.error(
'BcfProxy coult not connect to {0}'.format(self.controller['URI']))
else:
pass
@staticmethod
def make_hash(item):
''' hash the metadata in a sane way'''
h = hashlib.new('ripemd160')
pre_h = str()
post_h = None
for word in ['tenant', 'mac', 'segment', 'name', 'ip-address']:
pre_h = pre_h + str(item.get(str(word), 'missing'))
h.update(pre_h)
post_h = h.hexdigest()
return post_h
def handle_item(self, item):
''' perform an action based on rabbit item'''
self.logger.debug('handle_item: {0}:{1}'.format(item, type(item)))
itype = item[0]
ivalue = item[1]
ivalue = json.loads(ivalue)
self.logger.debug(
'handle_item: ivalue json: {0}:{1}'.format(ivalue, type(ivalue)))
if itype == 'poseidon.action.start_monitor':
for my_hash, my_dict in ivalue.iteritems():
if my_hash in self.new_endpoints:
v = self.new_endpoints.pop(my_hash)
self.logger.debug(
'removed {0} from new_endpoints'.format(v))
else:
self.logger.debug('could not find {0} in {1}'.format(
my_hash, self.new_endpoints))
self.logger.debug(
'mirroring :{0}'.format(my_dict['ip-address']))
self.logger.debug(
'mirroring[{0}]={1}'.format(my_hash, my_dict))
self.bcf.mirror_ip(my_dict['ip-address'])
self.mirroring[my_hash] = my_dict
if itype == 'poseidon.action.endpoint_shutdown':
self.logger.debug(
'endpoint_shutdown:{0}:{1}'.format(ivalue, type(ivalue)))
for my_hash, my_dict in ivalue.iteritems():
bad_ip = my_dict.get('ip-address')
if bad_ip is not None:
self.logger.debug(
'****** shutdown {0}:{1}'.format(bad_ip, ivalue))
self.bcf.shutdown_ip(bad_ip)
self.shutdown[my_hash] = my_dict
if itype == 'poseidon.action.stop_monitor':
self.logger.debug('stop_monitor:{0}:{1}'.format(itype, ivalue))
for my_hash, my_dict in ivalue.iteritems():
self.logger.debug('stop_monitor_dict:{0}'.format(my_dict))
my_ip = my_dict.get('ip-address')
if my_ip is not None:
self.logger.debug('***** shutting down {0}'.format(my_ip))
self.bcf.unmirror_ip(my_ip)
if my_hash in self.mirroring:
self.mirroring.pop(my_hash)
def get_rabbit_work(self):
'''get work item from queue if exists'''
# type , value
workfound = False
item = None
self.logger.debug('about to look for work')
try:
item = self.m_queue.get(False)
self.logger.debug('item:{0}'.format(item))
self.logger.debug('work found')
workfound = True
except Queue.Empty:
pass
self.logger.debug('done looking for work!')
if workfound:
self.handle_item(item)
return item
def find_new_machines(self, machines):
'''parse switch structure to find new machines added to network
since last call'''
if self.first_time:
self.first_time = False
# TODO db call to see if really need to run things
for machine in machines:
h = self.make_hash(machine)
module_logger.critical(
'adding address to known systems {0}'.format(machine))
self.prev_endpoints[h] = machine
else:
for machine in machines:
h = self.make_hash(machine)
if h not in self.prev_endpoints and h not in self.mirroring:
module_logger.critical(
'***** detected new address {0}'.format(machine))
self.new_endpoints[h] = machine
def print_state(self):
self.logger.debug('**************PREV*****************')
for my_hash, my_dict in self.prev_endpoints.iteritems():
self.logger.debug('P:{0}:{1}'.format(my_hash, my_dict))
self.logger.debug('**************NEW******************')
for my_hash, my_dict in self.new_endpoints.iteritems():
self.logger.debug('N:{0}:{1}'.format(my_hash, my_dict))
self.logger.debug('***********MIRRORING***************')
for my_hash, my_dict in self.mirroring.iteritems():
self.logger.debug('M:{0}:{1}'.format(my_hash, my_dict))
self.logger.debug('***********SHUTDOWN****************')
for my_hash, my_dict in self.shutdown.iteritems():
self.logger.debug('M:{0}:{1}'.format(my_hash, my_dict))
def send_new_machines(self):
'''send listing of new machines to main for decisions'''
for hashed, machine in self.new_endpoints.iteritems():
# TODO write findings to main
r_exchange = 'topic-poseidon-internal'
r_key = 'poseidon.action.new_machine'
r_msg = json.dumps({hashed: machine})
self.rabbit_channel_local.basic_publish(exchange=r_exchange,
routing_key=r_key,
body=r_msg)
def on_get(self, req, resp):
'''Handles Get requests'''
self.retval['service'] = self.owner.mod_name + ':' + self.mod_name
self.retval['times'] = self.times
self.retval['machines'] = None
self.retval['resp'] = 'bad'
current = None
parsed = None
machines = {}
try:
current = self.bcf.get_endpoints()
parsed = self.bcf.format_endpoints(current)
machines = parsed
except:
self.logger.error(
'Could not establish connection to {0}.'.format(self.controller['URI']))
self.retval['controller'] = 'Could not establish connection to {0}.'.format(
self.controller['URI'])
self.get_rabbit_work()
self.logger.debug('MACHINES:{0}'.format(machines))
self.find_new_machines(machines)
self.send_new_machines()
self.print_state()
self.retval['machines'] = parsed
self.retval['resp'] = 'ok'
# TODO change response to something reflecting success of traversal
self.times = self.times + 1
resp.body = json.dumps(self.retval)