def __init__(self, skip_rabbit): self.faucet_event = [] self.m_queue = queue.Queue() self.skip_rabbit = skip_rabbit self.logger = logger # get config options self.controller = Config().get_config() # timer class to call things periodically in own thread self.schedule = schedule # setup prometheus self.prom = Prometheus() try: self.prom.initialize_metrics() except Exception as e: # pragma: no cover self.logger.debug( 'Prometheus metrics are already initialized: {0}'.format( str(e))) Prometheus.start() # initialize sdnconnect self.s = SDNConnect() # retrieve endpoints from redis self.s.get_stored_endpoints() # set all retrieved endpoints to inactive at the start for endpoint in self.s.endpoints: if not endpoint.ignore: if endpoint.state != 'inactive': if endpoint.state == 'mirroring': endpoint.p_next_state = 'mirror' elif endpoint.state == 'reinvestigating': endpoint.p_next_state = 'reinvestigate' elif endpoint.state == 'queued': endpoint.p_next_state = 'queue' elif endpoint.state in ['known', 'abnormal']: endpoint.p_next_state = endpoint.state endpoint.endpoint_data['active'] = 0 endpoint.inactive() endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) # store changes to state self.s.store_endpoints() # schedule periodic scan of endpoints thread self.schedule.every(self.controller['scan_frequency']).seconds.do( partial(schedule_job_kickurl, func=self)) # schedule periodic reinvestigations thread self.schedule.every( self.controller['reinvestigation_frequency']).seconds.do( partial(schedule_job_reinvestigation, func=self)) # schedule all threads self.schedule_thread = threading.Thread(target=partial( schedule_thread_worker, schedule=self.schedule), name='st_worker')
def __init__(self, skip_rabbit, controller=None): self.faucet_event = [] self.m_queue = queue.Queue() self.job_queue = queue.Queue() self.skip_rabbit = skip_rabbit self.logger = logger self.rabbit_channel_connection_local = None self.rabbit_channel_connection_local_fa = None # get config options if controller is None: self.controller = Config().get_config() else: self.controller = controller # timer class to call things periodically in own thread self.schedule = schedule # setup prometheus self.prom = Prometheus() try: self.prom.initialize_metrics() except Exception as e: # pragma: no cover self.logger.debug( 'Prometheus metrics are already initialized: {0}'.format( str(e))) Prometheus.start() # initialize sdnconnect self.s = SDNConnect(self.controller) # schedule periodic scan of endpoints thread self.schedule.every(self.controller['scan_frequency']).seconds.do( self.schedule_job_kickurl) # schedule periodic reinvestigations thread self.schedule.every( self.controller['reinvestigation_frequency']).seconds.do( self.schedule_job_reinvestigation) # schedule all threads self.schedule_thread = threading.Thread(target=partial( schedule_thread_worker, schedule=self.schedule), name='st_worker')
class Monitor(object): def __init__(self, skip_rabbit): self.faucet_event = [] self.m_queue = queue.Queue() self.skip_rabbit = skip_rabbit self.logger = logger self.rabbit_channel_connection_local = None self.rabbit_channel_connection_local_fa = None # get config options self.controller = Config().get_config() # timer class to call things periodically in own thread self.schedule = schedule # setup prometheus self.prom = Prometheus() try: self.prom.initialize_metrics() except Exception as e: # pragma: no cover self.logger.debug( 'Prometheus metrics are already initialized: {0}'.format( str(e))) Prometheus.start() # initialize sdnconnect self.s = SDNConnect(self.controller) # schedule periodic scan of endpoints thread self.schedule.every(self.controller['scan_frequency']).seconds.do( partial(schedule_job_kickurl, schedule_func=self)) # schedule periodic reinvestigations thread self.schedule.every( self.controller['reinvestigation_frequency']).seconds.do( partial(schedule_job_reinvestigation, schedule_func=self)) # schedule all threads self.schedule_thread = threading.Thread(target=partial( schedule_thread_worker, schedule=self.schedule), name='st_worker') def format_rabbit_message(self, item): ''' read a message off the rabbit_q the message should be item = (routing_key,msg) ''' ret_val = {} routing_key, my_obj = item self.logger.debug('rabbit_message:{0}'.format(my_obj)) my_obj = json.loads(my_obj) self.logger.debug('routing_key:{0}'.format(routing_key)) remove_list = [] if routing_key == 'poseidon.algos.decider': self.logger.debug('decider value:{0}'.format(my_obj)) for name, message in my_obj.items(): endpoint = self.s.endpoints.get(name, None) if endpoint and message.get('plugin', None) == 'ncapture': endpoint.trigger('unknown') endpoint.p_next_state = None endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) if message.get('valid', False): ret_val.update(my_obj) else: ret_val = {} break elif routing_key == 'poseidon.action.ignore': for name in my_obj: endpoint = self.s.endpoints.get(name, None) if endpoint: endpoint.ignore = True elif routing_key == 'poseidon.action.clear.ignored': for name in my_obj: endpoint = self.s.endpoints.get(name, None) if endpoint: endpoint.ignore = False elif routing_key == 'poseidon.action.change': for name, state in my_obj: endpoint = self.s.endpoints.get(name, None) if endpoint: try: if state != 'mirror' and state != 'reinvestigate' and ( endpoint.state == 'mirroring' or endpoint.state == 'reinvestigating'): status = Actions(endpoint, self.s.sdnc).unmirror_endpoint() if not status: self.logger.warning( 'Unable to unmirror the endpoint: {0}'. format(endpoint.name)) endpoint.trigger(state) endpoint.p_next_state = None endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) if endpoint.state == 'mirroring' or endpoint.state == 'reinvestigating': status = Actions(endpoint, self.s.sdnc).mirror_endpoint() if status: try: self.s.r.hincrby('vent_plugin_counts', 'ncapture') except Exception as e: # pragma: no cover self.logger.error( 'Failed to update count of plugins because: {0}' .format(str(e))) else: self.logger.warning( 'Unable to mirror the endpoint: {0}'. format(endpoint.name)) except Exception as e: # pragma: no cover self.logger.error( 'Unable to change endpoint {0} because: {1}'. format(endpoint.name, str(e))) elif routing_key == 'poseidon.action.update_acls': for ip in my_obj: rules = my_obj[ip] endpoints = self.s.endpoints_by_ip(ip) if endpoints: endpoint = endpoints[0] try: status = Actions(endpoint, self.s.sdnc).update_acls( rules_file=self.controller['RULES_FILE'], endpoints=endpoints, force_apply_rules=rules) if not status: self.logger.warning( 'Unable to apply rules: {0} to endpoint: {1}'. format(rules, endpoint.name)) except Exception as e: self.logger.error( 'Unable to apply rules: {0} to endpoint: {1} because {2}' .format(rules, endpoint.name, str(e))) elif routing_key == 'poseidon.action.remove': remove_list = [name for name in my_obj] elif routing_key == 'poseidon.action.remove.ignored': remove_list = [ endpoint.name for endpoint in self.s.endpoints.values() if endpoint.ignore ] elif routing_key == 'poseidon.action.remove.inactives': remove_list = [ endpoint.name for endpoint in self.s.endpoints.values() if endpoint.state == 'inactive' ] elif routing_key == self.controller['FA_RABBIT_ROUTING_KEY']: self.logger.debug('FAUCET Event:{0}'.format(my_obj)) ret_val.update(my_obj) for endpoint_name in remove_list: if endpoint_name in self.s.endpoints: del self.s.endpoints[endpoint_name] return ret_val def process(self): global CTRL_C signal.signal(signal.SIGINT, partial(self.signal_handler)) while not CTRL_C['STOP']: time.sleep(1) found_work, item = self.get_q_item() ml_returns = {} if found_work and item[0] == self.controller[ 'FA_RABBIT_ROUTING_KEY']: self.faucet_event.append(self.format_rabbit_message(item)) self.logger.debug('Faucet event: {0}'.format( self.faucet_event)) elif found_work: msg = self.format_rabbit_message(item) if 'data' in msg: ml_returns = msg['data'] if ml_returns: self.logger.info('ML results: {0}'.format(ml_returns)) extras = deepcopy(ml_returns) # process results from ml output and update impacted endpoints for ep in self.s.endpoints.values(): if ep.name in ml_returns: del extras[ep.name] if ep.name in ml_returns and 'valid' in ml_returns[ ep.name] and not ep.ignore: if ep.state in ['mirroring', 'reinvestigating']: status = Actions(ep, self.s.sdnc).unmirror_endpoint() if not status: self.logger.warning( 'Unable to unmirror the endpoint: {0}'. format(ep.name)) if ml_returns[ep.name]['valid']: ml_decision = None if 'decisions' in ml_returns[ ep.name] and 'behavior' in ml_returns[ ep.name]['decisions']: ml_decision = ml_returns[ ep.name]['decisions']['behavior'] if ml_decision == 'normal': ep.known() else: ep.abnormal() else: ep.unknown() ep.p_prev_states.append((ep.state, int(time.time()))) extra_machines = [] self.logger.debug('extra devices: {0}'.format(extras)) for device in extras: if device['valid']: extra_machine = { 'mac': device['source_mac'], 'segment': NO_DATA, 'port': NO_DATA, 'tenant': NO_DATA, 'active': 0, 'name': None } try: source_ip = ipaddress.ip_address( device['source_ip']) except ValueError: source_ip = None if source_ip: extra_machine['ipv%u' % source_ip.version] = str(source_ip) extra_machines.append(extra_machine) self.s.find_new_machines(extra_machines) queued_endpoints = [ endpoint for endpoint in self.s.endpoints.values() if not endpoint.ignore and endpoint.state == 'queued' and endpoint.p_next_state != 'inactive' ] self.s.investigations = len([ endpoint for endpoint in self.s.endpoints.values() if endpoint.state in ['mirroring', 'reinvestigating'] ]) # mirror things in the order they got added to the queue queued_endpoints = sorted(queued_endpoints, key=lambda x: x.p_prev_states[-1][1]) investigation_budget = max( self.controller['max_concurrent_reinvestigations'] - self.s.investigations, 0) self.logger.debug( 'investigations {0}, budget {1}, queued {2}'.format( str(self.s.investigations), str(investigation_budget), str(len(queued_endpoints)))) for endpoint in queued_endpoints[:investigation_budget]: endpoint.trigger(endpoint.p_next_state) endpoint.p_next_state = None endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) status = Actions(endpoint, self.s.sdnc).mirror_endpoint() if status: try: if self.s.r: self.s.r.hincrby('vent_plugin_counts', 'ncapture') except Exception as e: # pragma: no cover self.logger.error( 'Failed to update count of plugins because: {0}'. format(str(e))) else: self.logger.warning( 'Unable to mirror the endpoint: {0}'.format( endpoint.name)) for endpoint in self.s.endpoints.values(): if not endpoint.ignore: if self.s.sdnc: if endpoint.state == 'unknown': endpoint.p_next_state = 'mirror' endpoint.queue() endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) elif endpoint.state in [ 'mirroring', 'reinvestigating' ]: cur_time = int(time.time()) # timeout after 2 times the reinvestigation frequency # in case something didn't report back, put back in an # unknown state if cur_time - endpoint.p_prev_states[-1][ 1] > 2 * self.controller[ 'reinvestigation_frequency']: self.logger.debug( 'timing out: {0} and setting to unknown'. format(endpoint.name)) status = Actions( endpoint, self.s.sdnc).unmirror_endpoint() if not status: self.logger.warning( 'Unable to unmirror the endpoint: {0}'. format(endpoint.name)) endpoint.unknown() endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) else: if endpoint.state != 'known': endpoint.known() self.s.store_endpoints() return def get_q_item(self): ''' attempt to get a work item from the queue m_queue -> (routing_key, body) a read from get_q_item should be of the form (boolean,(routing_key, body)) ''' found_work = False item = None global CTRL_C if not CTRL_C['STOP']: try: item = self.m_queue.get(False) found_work = True self.m_queue.task_done() except queue.Empty: # pragma: no cover pass return (found_work, item) def shutdown(self): ''' gracefully shut down. ''' self.s.clear_filters() for job in self.schedule.jobs: self.logger.debug('shutdown :{0}'.format(job)) self.schedule.cancel_job(job) if self.rabbit_channel_connection_local: self.rabbit_channel_connection_local.close() if self.rabbit_channel_connection_local_fa: self.rabbit_channel_connection_local_fa.close() self.logger.debug('SHUTTING DOWN') self.logger.debug('EXITING') sys.exit() def signal_handler(self, signal, frame): ''' hopefully eat a CTRL_C and signal system shutdown ''' global CTRL_C CTRL_C['STOP'] = True self.logger.debug('CTRL-C: {0}'.format(CTRL_C)) try: self.shutdown() except Exception as e: # pragma: no cover self.logger.debug( 'Failed to handle signal properly because: {0}'.format(str(e)))
class Monitor(object): def __init__(self, skip_rabbit): self.faucet_event = [] self.m_queue = queue.Queue() self.skip_rabbit = skip_rabbit self.logger = logger # get config options self.controller = Config().get_config() # timer class to call things periodically in own thread self.schedule = schedule # setup prometheus self.prom = Prometheus() try: self.prom.initialize_metrics() except Exception as e: # pragma: no cover self.logger.debug( 'Prometheus metrics are already initialized: {0}'.format( str(e))) Prometheus.start() # initialize sdnconnect self.s = SDNConnect() # cleanup any old filters if isinstance(self.s.sdnc, FaucetProxy): Parser().clear_mirrors(self.controller['CONFIG_FILE']) elif isinstance(self.s.sdnc, BcfProxy): self.s.sdnc.remove_filter_rules() # retrieve endpoints from redis self.s.get_stored_endpoints() # set all retrieved endpoints to inactive at the start for endpoint in self.s.endpoints: if not endpoint.ignore: if endpoint.state != 'inactive': if endpoint.state == 'mirroring': endpoint.p_next_state = 'mirror' elif endpoint.state == 'reinvestigating': endpoint.p_next_state = 'reinvestigate' elif endpoint.state == 'queued': endpoint.p_next_state = 'queue' elif endpoint.state in ['known', 'abnormal']: endpoint.p_next_state = endpoint.state endpoint.endpoint_data['active'] = 0 endpoint.inactive() endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) # store changes to state self.s.store_endpoints() # schedule periodic scan of endpoints thread self.schedule.every(self.controller['scan_frequency']).seconds.do( partial(schedule_job_kickurl, func=self)) # schedule periodic reinvestigations thread self.schedule.every( self.controller['reinvestigation_frequency']).seconds.do( partial(schedule_job_reinvestigation, func=self)) # schedule all threads self.schedule_thread = threading.Thread(target=partial( schedule_thread_worker, schedule=self.schedule), name='st_worker') def format_rabbit_message(self, item): ''' read a message off the rabbit_q the message should be item = (routing_key,msg) ''' ret_val = {} routing_key, my_obj = item self.logger.debug('rabbit_message:{0}'.format(my_obj)) my_obj = json.loads(my_obj) self.logger.debug('routing_key:{0}'.format(routing_key)) if routing_key == 'poseidon.algos.decider': self.logger.debug('decider value:{0}'.format(my_obj)) # TODO if valid response then send along otherwise nothing for key in my_obj: ret_val[key] = my_obj[key] elif routing_key == 'poseidon.action.ignore': for name in my_obj: for endpoint in self.s.endpoints: if name == endpoint.name: endpoint.ignore = True elif routing_key == 'poseidon.action.clear.ignored': for name in my_obj: for endpoint in self.s.endpoints: if name == endpoint.name: endpoint.ignore = False elif routing_key == 'poseidon.action.change': for name, state in my_obj: for endpoint in self.s.endpoints: if name == endpoint.name: try: if state != 'mirror' and state != 'reinvestigate' and ( endpoint.state == 'mirroring' or endpoint.state == 'reinvestigating'): status = Actions( endpoint, self.s.sdnc).unmirror_endpoint() if not status: self.logger.warning( 'Unable to unmirror the endpoint: {0}'. format(endpoint.name)) endpoint.trigger(state) endpoint.p_next_state = None endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) if endpoint.state == 'mirroring' or endpoint.state == 'reinvestigating': status = Actions( endpoint, self.s.sdnc).mirror_endpoint() if status: try: self.r.hincrby('vent_plugin_counts', 'ncapture') except Exception as e: # pragma: no cover self.logger.error( 'Failed to update count of plugins because: {0}' .format(str(e))) else: self.logger.warning( 'Unable to mirror the endpoint: {0}'. format(endpoint.name)) except Exception as e: # pragma: no cover self.logger.error( 'Unable to change endpoint {0} because: {1}'. format(endpoint.name, str(e))) elif routing_key == 'poseidon.action.remove': remove_list = [] for name in my_obj: for endpoint in self.s.endpoints: if name == endpoint.name: remove_list.append(endpoint) for endpoint in remove_list: self.s.endpoints.remove(endpoint) elif routing_key == 'poseidon.action.remove.ignored': remove_list = [] for endpoint in self.s.endpoints: if endpoint.ignore: remove_list.append(endpoint) for endpoint in remove_list: self.s.endpoints.remove(endpoint) elif routing_key == 'poseidon.action.remove.inactives': remove_list = [] for endpoint in self.s.endpoints: if endpoint.state == 'inactive': remove_list.append(endpoint) for endpoint in remove_list: self.s.endpoints.remove(endpoint) elif routing_key == self.controller['FA_RABBIT_ROUTING_KEY']: self.logger.debug('FAUCET Event:{0}'.format(my_obj)) for key in my_obj: ret_val[key] = my_obj[key] return ret_val def process(self): global CTRL_C signal.signal(signal.SIGINT, partial(self.signal_handler)) while not CTRL_C['STOP']: time.sleep(1) found_work, item = self.get_q_item() ml_returns = {} if found_work and item[0] == self.controller[ 'FA_RABBIT_ROUTING_KEY']: self.faucet_event.append(self.format_rabbit_message(item)) self.logger.debug('Faucet event: {0}'.format( self.faucet_event)) elif found_work: ml_returns = self.format_rabbit_message(item) if ml_returns: self.logger.info('ML results: {0}'.format(ml_returns)) extras = deepcopy(ml_returns) # process results from ml output and update impacted endpoints for ep in self.s.endpoints: if ep.name in ml_returns: del extras[ep.name] if ep.name in ml_returns and 'valid' in ml_returns[ ep.name] and not ep.ignore: if ep.state in ['mirroring', 'reinvestigating']: status = Actions(ep, self.s.sdnc).unmirror_endpoint() if not status: self.logger.warning( 'Unable to unmirror the endpoint: {0}'. format(ep.name)) self.s.investigations -= 1 if ml_returns[ep.name]['valid']: ml_decision = None if 'decisions' in ml_returns[ ep.name] and 'behavior' in ml_returns[ ep.name]['decisions']: ml_decision = ml_returns[ ep.name]['decisions']['behavior'] if ml_decision == 'normal': ep.known() else: ep.abnormal() else: ep.unknown() ep.p_prev_states.append((ep.state, int(time.time()))) extra_machines = [] for device in extras: if extras[device]['valid']: extra_machine = { 'mac': extras[device]['source_mac'], 'segment': 'NO DATA', 'port': 'NO DATA', 'tenant': 'NO DATA', 'active': 0, 'name': None } if ':' in extras[device]['source_ip']: extra_machine['ipv6'] = extras[device]['source_ip'] extra_machine['ipv4'] = 0 else: extra_machine['ipv4'] = extras[device]['source_ip'] extra_machine['ipv6'] = 0 extra_machines.append(extra_machine) self.s.find_new_machines(extra_machines) # mirror things in the order they got added to the queue queued_endpoints = [] for endpoint in self.s.endpoints: if not endpoint.ignore: if endpoint.state == 'queued': queued_endpoints.append( (endpoint.name, endpoint.p_prev_states[-1][1])) queued_endpoints = sorted(queued_endpoints, key=lambda x: x[1]) for ep in queued_endpoints: for endpoint in self.s.endpoints: if ep[0] == endpoint.name: if self.s.investigations < self.controller[ 'max_concurrent_reinvestigations']: self.s.investigations += 1 endpoint.trigger(endpoint.p_next_state) endpoint.p_next_state = None endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) status = Actions(endpoint, self.s.sdnc).mirror_endpoint() if status: try: self.s.r.hincrby('vent_plugin_counts', 'ncapture') except Exception as e: # pragma: no cover self.logger.error( 'Failed to update count of plugins because: {0}' .format(str(e))) else: self.logger.warning( 'Unable to mirror the endpoint: {0}'. format(endpoint.name)) for endpoint in self.s.endpoints: if not endpoint.ignore: if self.s.sdnc: if endpoint.state == 'unknown': endpoint.p_next_state = 'mirror' endpoint.queue() endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) elif endpoint.state in [ 'mirroring', 'reinvestigating' ]: cur_time = int(time.time()) # timeout after 2 times the reinvestigation frequency # in case something didn't report back, put back in an # unknown state if cur_time - endpoint.p_prev_states[-1][ 1] > 2 * self.controller[ 'reinvestigation_frequency']: status = Actions( endpoint, self.s.sdnc).unmirror_endpoint() if not status: self.logger.warning( 'Unable to unmirror the endpoint: {0}'. format(endpoint.name)) endpoint.unknown() self.s.investigations -= 1 endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) else: if endpoint.state != 'known': endpoint.known() return def get_q_item(self): ''' attempt to get a work item from the queue m_queue -> (routing_key, body) a read from get_q_item should be of the form (boolean,(routing_key, body)) ''' found_work = False item = None global CTRL_C if not CTRL_C['STOP']: try: item = self.m_queue.get(False) found_work = True self.m_queue.task_done() except queue.Empty: # pragma: no cover pass return (found_work, item) def signal_handler(self, signal, frame): ''' hopefully eat a CTRL_C and signal system shutdown ''' global CTRL_C if isinstance(self.s.sdnc, FaucetProxy): Parser().clear_mirrors(self.controller['CONFIG_FILE']) elif isinstance(self.s.sdnc, BcfProxy): self.logger.debug('removing bcf filter rules') retval = self.s.sdnc.remove_filter_rules() self.logger.debug('removed filter rules: {0}'.format(retval)) CTRL_C['STOP'] = True self.logger.debug('CTRL-C: {0}'.format(CTRL_C)) try: for job in self.schedule.jobs: self.logger.debug('CTRLC:{0}'.format(job)) self.schedule.cancel_job(job) self.rabbit_channel_connection_local.close() self.rabbit_channel_connection_local_fa.close() self.logger.debug('SHUTTING DOWN') self.logger.debug('EXITING') sys.exit() except Exception as e: # pragma: no cover self.logger.debug( 'Failed to handle signal properly because: {0}'.format(str(e)))
class Monitor: def __init__(self, skip_rabbit, controller=None): self.faucet_event = [] self.m_queue = queue.Queue() self.job_queue = queue.Queue() self.skip_rabbit = skip_rabbit self.logger = logger self.rabbit_channel_connection_local = None self.rabbit_channel_connection_local_fa = None # get config options if controller is None: self.controller = Config().get_config() else: self.controller = controller # timer class to call things periodically in own thread self.schedule = schedule # setup prometheus self.prom = Prometheus() try: self.prom.initialize_metrics() except Exception as e: # pragma: no cover self.logger.debug( 'Prometheus metrics are already initialized: {0}'.format( str(e))) Prometheus.start() # initialize sdnconnect self.s = SDNConnect(self.controller) # schedule periodic scan of endpoints thread self.schedule.every(self.controller['scan_frequency']).seconds.do( self.schedule_job_kickurl) # schedule periodic reinvestigations thread self.schedule.every( self.controller['reinvestigation_frequency']).seconds.do( self.schedule_job_reinvestigation) # schedule all threads self.schedule_thread = threading.Thread(target=partial( schedule_thread_worker, schedule=self.schedule), name='st_worker') def _update_metrics(self): self.logger.debug('updating metrics') try: # get current state req = requests.get('http://poseidon-api:8000/v1/network_full', timeout=10) # send results to prometheus hosts = req.json()['dataset'] self.prom.update_metrics(hosts) except (requests.exceptions.ConnectionError, Exception) as e: # pragma: no cover self.logger.error( 'Unable to get current state and send it to Prometheus because: {0}' .format(str(e))) def job_kickurl(self): self.s.check_endpoints(messages=self.faucet_event) del self.faucet_event[:] self._update_metrics() def job_reinvestigation(self): ''' put endpoints into the reinvestigation state if possible ''' def trigger_reinvestigation(candidates): # get random order of things that are known for _ in range(self.controller['max_concurrent_reinvestigations'] - self.s.investigations): if len(candidates) > 0: chosen = candidates.pop() self.logger.info( 'Starting reinvestigation on: {0} {1}'.format( chosen.name, chosen.state)) chosen.reinvestigate() # pytype: disable=attribute-error chosen.p_prev_states.append( (chosen.state, int(time.time()))) self.s.mirror_endpoint(chosen) candidates = [ endpoint for endpoint in self.s.endpoints.values() if endpoint.state in ['queued'] ] if len(candidates) == 0: # if no queued endpoints, then known and abnormal are candidates candidates = [ endpoint for endpoint in self.s.endpoints.values() if endpoint.state in ['known', 'abnormal'] ] if len(candidates) > 0: random.shuffle(candidates) if self.s.sdnc: trigger_reinvestigation(candidates) def queue_job(self, job): if self.job_queue.qsize() < 2: self.job_queue.put(job) def schedule_job_kickurl(self): self.queue_job(self.job_kickurl) def schedule_job_reinvestigation(self): self.queue_job(self.job_reinvestigation) def update_routing_key_time(self, routing_key): self.prom.prom_metrics['last_rabbitmq_routing_key_time'].labels( routing_key=routing_key).set(time.time()) def format_rabbit_message(self, item): ''' read a message off the rabbit_q the message should be item = (routing_key,msg) ''' routing_key, my_obj = item self.logger.debug('routing_key: {0} rabbit_message: {1}'.format( routing_key, my_obj)) my_obj = json.loads(my_obj) def handler_algos_decider(my_obj): self.logger.debug('decider value:{0}'.format(my_obj)) data = my_obj.get('data', None) results = my_obj.get('results', {}) tool = results.get('tool', None) if isinstance(data, dict): if tool == 'p0f': if self.s.prc.store_p0f_result(data): return (data, None) elif tool == 'networkml': self.s.prc.store_tool_result(my_obj, 'networkml') for name, message in data.items(): endpoint = self.s.endpoints.get(name, None) if endpoint: self.logger.debug( 'processing networkml results for %s', name) self.s.unmirror_endpoint(endpoint) # pytype: disable=attribute-error endpoint.trigger('unknown') endpoint.p_next_state = None endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) if message.get('valid', False): return (data, None) break else: self.logger.debug( 'endpoint %s from networkml not found', name) return ({}, None) def handler_action_ignore(my_obj): for name in my_obj: endpoint = self.s.endpoints.get(name, None) if endpoint: endpoint.ignore = True return ({}, None) def handler_action_clear_ignored(my_obj): for name in my_obj: endpoint = self.s.endpoints.get(name, None) if endpoint: endpoint.ignore = False return ({}, None) def handler_action_change(my_obj): for name, state in my_obj: endpoint = self.s.endpoints.get(name, None) if endpoint: try: if (state != 'mirror' and state != 'reinvestigate' and (endpoint.state == 'mirroring' or endpoint.state == 'reinvestigating')): self.s.unmirror_endpoint(endpoint) # pytype: disable=attribute-error endpoint.trigger(state) endpoint.p_next_state = None endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) if endpoint.state == 'mirroring' or endpoint.state == 'reinvestigating': self.s.mirror_endpoint(endpoint) except Exception as e: # pragma: no cover self.logger.error( 'Unable to change endpoint {0} because: {1}'. format(endpoint.name, str(e))) return ({}, None) def handler_action_update_acls(my_obj): for ip in my_obj: rules = my_obj[ip] endpoints = self.s.endpoints_by_ip(ip) if endpoints: endpoint = endpoints[0] try: status = Actions(endpoint, self.s.sdnc).update_acls( rules_file=self.controller['RULES_FILE'], endpoints=endpoints, force_apply_rules=rules) if not status: self.logger.warning( 'Unable to apply rules: {0} to endpoint: {1}'. format(rules, endpoint.name)) except Exception as e: self.logger.error( 'Unable to apply rules: {0} to endpoint: {1} because {2}' .format(rules, endpoint.name, str(e))) return ({}, None) def handler_action_remove(my_obj): remove_list = [name for name in my_obj] return ({}, remove_list) def handler_action_remove_ignored(_my_obj): remove_list = [ endpoint.name for endpoint in self.s.endpoints.values() if endpoint.ignore ] return ({}, remove_list) def handler_action_remove_inactives(_my_obj): remove_list = [ endpoint.name for endpoint in self.s.endpoints.values() if endpoint.state == 'inactive' ] return ({}, remove_list) def handler_faucet_event(my_obj): if self.s and self.s.sdnc: if not self.s.sdnc.ignore_event(my_obj): self.faucet_event.append(my_obj) return (my_obj, None) return ({}, None) handlers = { 'poseidon.algos.decider': handler_algos_decider, 'poseidon.action.ignore': handler_action_ignore, 'poseidon.action.clear.ignored': handler_action_clear_ignored, 'poseidon.action.change': handler_action_change, 'poseidon.action.update_acls': handler_action_update_acls, 'poseidon.action.remove': handler_action_remove, 'poseidon.action.remove.ignored': handler_action_remove_ignored, 'poseidon.action.remove.inactives': handler_action_remove_inactives, self.controller['FA_RABBIT_ROUTING_KEY']: handler_faucet_event, } handler = handlers.get(routing_key, None) if handler is None: self.logger.error( 'no handler for routing_key {0}'.format(routing_key)) else: ret_val, remove_list = handler(my_obj) self.update_routing_key_time(routing_key) if remove_list: for endpoint_name in remove_list: if endpoint_name in self.s.endpoints: del self.s.endpoints[endpoint_name] return (ret_val, True) return ({}, False) def schedule_mirroring(self): queued_endpoints = [ endpoint for endpoint in self.s.endpoints.values() if not endpoint.ignore and endpoint.state == 'queued' and endpoint.p_next_state != 'inactive' ] self.s.investigations = len([ endpoint for endpoint in self.s.endpoints.values() if endpoint.state in ['mirroring', 'reinvestigating'] ]) # mirror things in the order they got added to the queue queued_endpoints = sorted(queued_endpoints, key=lambda x: x.p_prev_states[-1][1]) investigation_budget = max( self.controller['max_concurrent_reinvestigations'] - self.s.investigations, 0) self.logger.debug('investigations {0}, budget {1}, queued {2}'.format( str(self.s.investigations), str(investigation_budget), str(len(queued_endpoints)))) for endpoint in queued_endpoints[:investigation_budget]: # pytype: disable=attribute-error endpoint.trigger(endpoint.p_next_state) endpoint.p_next_state = None endpoint.p_prev_states.append((endpoint.state, int(time.time()))) self.s.mirror_endpoint(endpoint) for endpoint in self.s.endpoints.values(): if not endpoint.ignore: if self.s.sdnc: if endpoint.state == 'unknown': endpoint.p_next_state = 'mirror' endpoint.queue() # pytype: disable=attribute-error endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) elif endpoint.state in ['mirroring', 'reinvestigating']: cur_time = int(time.time()) # timeout after 2 times the reinvestigation frequency # in case something didn't report back, put back in an # unknown state if cur_time - endpoint.p_prev_states[-1][ 1] > 2 * self.controller[ 'reinvestigation_frequency']: self.logger.debug( 'timing out: {0} and setting to unknown'. format(endpoint.name)) self.s.unmirror_endpoint(endpoint) endpoint.unknown() # pytype: disable=attribute-error endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) else: if endpoint.state != 'known': endpoint.known() # pytype: disable=attribute-error def schedule_coprocessing(self): queued_endpoints = [ endpoint for endpoint in self.s.endpoints.values() if not endpoint.copro_ignore and endpoint.copro_state == 'copro_queued' ] # pytype: disable=attribute-error self.s.coprocessing = len([ endpoint for endpoint in self.s.endpoints.values() if endpoint.copro_state in ['copro_coprocessing'] ]) # mirror things in the order they got added to the queue queued_endpoints = sorted(queued_endpoints, key=lambda x: x.p_prev_copro_states[-1][1]) coprocessing_budget = max( self.controller['max_concurrent_coprocessing'] - self.s.coprocessing, 0) self.logger.debug('coprocessing {0}, budget {1}, queued {2}'.format( str(self.s.coprocessing), str(coprocessing_budget), str(len(queued_endpoints)))) for endpoint in queued_endpoints[:coprocessing_budget]: # pytype: disable=attribute-error endpoint.trigger(endpoint.p_next_copro_state) endpoint.p_next_copro_state = None # pytype: disable=attribute-error endpoint.p_prev_copro_states.append( # pytype: disable=attribute-error (endpoint.copro_state, int(time.time()))) self.s.coprocess_endpoint(endpoint) for endpoint in self.s.endpoints.values(): if not endpoint.copro_ignore: # pytype: disable=attribute-error if self.s.sdnc: if endpoint.copro_state == 'copro_unknown': # pytype: disable=attribute-error endpoint.p_next_copro_state = 'copro_coprocessing' endpoint.copro_queue() # pytype: disable=attribute-error endpoint.p_prev_copro_states.append( # pytype: disable=attribute-error (endpoint.copro_state, int(time.time()))) # pytype: disable=attribute-error elif endpoint.copro_state in ['copro_coprocessing']: cur_time = int(time.time()) # timeout after 2 times the reinvestigation frequency # in case something didn't report back, put back in an # unknown state # pytype: disable=attribute-error if cur_time - endpoint.p_prev_copro_states[-1][ 1] > 2 * self.controller[ 'coprocessing_frequency']: self.logger.debug( 'timing out: {0} and setting to unknown'. format(endpoint.name)) self.s.uncoprocess_endpoint(endpoint) endpoint.copro_unknown() # pytype: disable=attribute-error endpoint.p_prev_copro_states.append( # pytype: disable=attribute-error (endpoint.copro_state, int(time.time()))) # pytype: disable=attribute-error else: if endpoint.state != 'copro_nominal': endpoint.copro_nominal() # pytype: disable=attribute-error def process(self): global CTRL_C signal.signal(signal.SIGINT, partial(self.signal_handler)) while not CTRL_C['STOP']: while True: found_work, rabbit_msg = self.get_q_item(self.m_queue, timeout=0) if not found_work: break self.format_rabbit_message(rabbit_msg) self.s.refresh_endpoints() found_work, schedule_func = self.get_q_item(self.job_queue) if found_work and callable(schedule_func): self.logger.info('calling %s', schedule_func) start_time = time.time() schedule_func() self.logger.debug('%s done (%.1f sec)' % (schedule_func, time.time() - start_time)) self.schedule_mirroring() self.s.refresh_endpoints() def get_q_item(self, q, timeout=1): ''' attempt to get a work item from the queue m_queue -> (routing_key, body) a read from get_q_item should be of the form (boolean,(routing_key, body)) ''' global CTRL_C if not CTRL_C['STOP']: try: if timeout: return (True, q.get(True, timeout=timeout)) return (True, q.get_nowait()) except queue.Empty: # pragma: no cover pass return (False, None) def shutdown(self): ''' gracefully shut down. ''' self.s.clear_filters() for job in self.schedule.jobs: self.logger.debug('shutdown :{0}'.format(job)) self.schedule.cancel_job(job) if self.rabbit_channel_connection_local: self.rabbit_channel_connection_local.close() if self.rabbit_channel_connection_local_fa: self.rabbit_channel_connection_local_fa.close() self.logger.debug('SHUTTING DOWN') self.logger.debug('EXITING') sys.exit() def signal_handler(self, _signal, _frame): ''' hopefully eat a CTRL_C and signal system shutdown ''' global CTRL_C CTRL_C['STOP'] = True self.logger.debug('CTRL-C: {0}'.format(CTRL_C)) try: self.shutdown() except Exception as e: # pragma: no cover self.logger.debug( 'Failed to handle signal properly because: {0}'.format(str(e)))
class Monitor: def __init__(self, skip_rabbit): self.faucet_event = [] self.m_queue = queue.Queue() self.skip_rabbit = skip_rabbit self.logger = logger self.rabbit_channel_connection_local = None self.rabbit_channel_connection_local_fa = None # get config options self.controller = Config().get_config() # timer class to call things periodically in own thread self.schedule = schedule # setup prometheus self.prom = Prometheus() try: self.prom.initialize_metrics() except Exception as e: # pragma: no cover self.logger.debug( 'Prometheus metrics are already initialized: {0}'.format( str(e))) Prometheus.start() # initialize sdnconnect self.s = SDNConnect(self.controller) # schedule periodic scan of endpoints thread self.schedule.every(self.controller['scan_frequency']).seconds.do( partial(schedule_job_kickurl, schedule_func=self)) # schedule periodic reinvestigations thread self.schedule.every( self.controller['reinvestigation_frequency']).seconds.do( partial(schedule_job_reinvestigation, schedule_func=self)) # schedule all threads self.schedule_thread = threading.Thread(target=partial( schedule_thread_worker, schedule=self.schedule), name='st_worker') def update_routing_key_time(self, routing_key): self.prom.prom_metrics['last_rabbitmq_routing_key_time'].labels( routing_key=routing_key).set(time.time()) def format_rabbit_message(self, item): ''' read a message off the rabbit_q the message should be item = (routing_key,msg) ''' routing_key, my_obj = item self.logger.debug('routing_key: {0} rabbit_message: {1}'.format( routing_key, my_obj)) my_obj = json.loads(my_obj) def handler_algos_decider(my_obj): self.logger.debug('decider value:{0}'.format(my_obj)) for name, message in my_obj.items(): endpoint = self.s.endpoints.get(name, None) if endpoint and message.get('plugin', None) == 'ncapture': endpoint.trigger('unknown') endpoint.p_next_state = None endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) if message.get('valid', False): return (my_obj, None) break return ({}, None) def handler_action_ignore(my_obj): for name in my_obj: endpoint = self.s.endpoints.get(name, None) if endpoint: endpoint.ignore = True return ({}, None) def handler_action_clear_ignored(my_obj): for name in my_obj: endpoint = self.s.endpoints.get(name, None) if endpoint: endpoint.ignore = False return ({}, None) def handler_action_change(my_obj): for name, state in my_obj: endpoint = self.s.endpoints.get(name, None) if endpoint: try: if (state != 'mirror' and state != 'reinvestigate' and (endpoint.state == 'mirroring' or endpoint.state == 'reinvestigating')): self.s.unmirror_endpoint(endpoint) endpoint.trigger(state) endpoint.p_next_state = None endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) if endpoint.state == 'mirroring' or endpoint.state == 'reinvestigating': self.s.mirror_endpoint(endpoint) except Exception as e: # pragma: no cover self.logger.error( 'Unable to change endpoint {0} because: {1}'. format(endpoint.name, str(e))) return ({}, None) def handler_action_update_acls(my_obj): for ip in my_obj: rules = my_obj[ip] endpoints = self.s.endpoints_by_ip(ip) if endpoints: endpoint = endpoints[0] try: status = Actions(endpoint, self.s.sdnc).update_acls( rules_file=self.controller['RULES_FILE'], endpoints=endpoints, force_apply_rules=rules) if not status: self.logger.warning( 'Unable to apply rules: {0} to endpoint: {1}'. format(rules, endpoint.name)) except Exception as e: self.logger.error( 'Unable to apply rules: {0} to endpoint: {1} because {2}' .format(rules, endpoint.name, str(e))) return ({}, None) def handler_action_remove(my_obj): remove_list = [name for name in my_obj] return ({}, remove_list) def handler_action_remove_ignored(_my_obj): remove_list = [ endpoint.name for endpoint in self.s.endpoints.values() if endpoint.ignore ] return ({}, remove_list) def handler_action_remove_inactives(_my_obj): remove_list = [ endpoint.name for endpoint in self.s.endpoints.values() if endpoint.state == 'inactive' ] return ({}, remove_list) def handler_faucet_event(my_obj): self.logger.debug('FAUCET Event:{0}'.format(my_obj)) self.faucet_event.append(my_obj) return (my_obj, None) handlers = { 'poseidon.algos.decider': handler_algos_decider, 'poseidon.action.ignore': handler_action_ignore, 'poseidon.action.clear.ignored': handler_action_clear_ignored, 'poseidon.action.change': handler_action_change, 'poseidon.action.update_acls': handler_action_update_acls, 'poseidon.action.remove': handler_action_remove, 'poseidon.action.remove.ignored': handler_action_remove_ignored, 'poseidon.action.remove.inactives': handler_action_remove_inactives, self.controller['FA_RABBIT_ROUTING_KEY']: handler_faucet_event, } handler = handlers.get(routing_key, None) if handler is None: self.logger.error( 'no handler for routing_key {0}'.format(routing_key)) else: ret_val, remove_list = handler(my_obj) self.update_routing_key_time(routing_key) if remove_list: for endpoint_name in remove_list: if endpoint_name in self.s.endpoints: del self.s.endpoints[endpoint_name] return (ret_val, True) return ({}, False) def schedule_mirroring(self): queued_endpoints = [ endpoint for endpoint in self.s.endpoints.values() if not endpoint.ignore and endpoint.state == 'queued' and endpoint.p_next_state != 'inactive' ] self.s.investigations = len([ endpoint for endpoint in self.s.endpoints.values() if endpoint.state in ['mirroring', 'reinvestigating'] ]) # mirror things in the order they got added to the queue queued_endpoints = sorted(queued_endpoints, key=lambda x: x.p_prev_states[-1][1]) investigation_budget = max( self.controller['max_concurrent_reinvestigations'] - self.s.investigations, 0) self.logger.debug('investigations {0}, budget {1}, queued {2}'.format( str(self.s.investigations), str(investigation_budget), str(len(queued_endpoints)))) for endpoint in queued_endpoints[:investigation_budget]: endpoint.trigger(endpoint.p_next_state) endpoint.p_next_state = None endpoint.p_prev_states.append((endpoint.state, int(time.time()))) self.s.mirror_endpoint(endpoint) for endpoint in self.s.endpoints.values(): if not endpoint.ignore: if self.s.sdnc: if endpoint.state == 'unknown': endpoint.p_next_state = 'mirror' endpoint.queue() endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) elif endpoint.state in ['mirroring', 'reinvestigating']: cur_time = int(time.time()) # timeout after 2 times the reinvestigation frequency # in case something didn't report back, put back in an # unknown state if cur_time - endpoint.p_prev_states[-1][ 1] > 2 * self.controller[ 'reinvestigation_frequency']: self.logger.debug( 'timing out: {0} and setting to unknown'. format(endpoint.name)) self.s.unmirror_endpoint(endpoint) endpoint.unknown() endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) else: if endpoint.state != 'known': endpoint.known() def schedule_coprocessing(self): queued_endpoints = [ endpoint for endpoint in self.s.endpoints.values() if not endpoint.copro_ignore and endpoint.copro_state == 'queued' ] self.s.coprocessing = len([ endpoint for endpoint in self.s.endpoints.values() if endpoint.copro_state in ['coprocessing'] ]) # mirror things in the order they got added to the queue queued_endpoints = sorted(queued_endpoints, key=lambda x: x.p_prev_copro_states[-1][1]) coprocessing_budget = max( self.controller['max_concurrent_coprocessing'] - self.s.coprocessing, 0) self.logger.debug('coprocessing {0}, budget {1}, queued {2}'.format( str(self.s.coprocessing), str(coprocessing_budget), str(len(queued_endpoints)))) for endpoint in queued_endpoints[:coprocessing_budget]: endpoint.trigger(endpoint.p_next_copro_state) endpoint.p_next_copro_state = None endpoint.p_prev_copro_states.append( (endpoint.copro_state, int(time.time()))) self.s.coprocess_endpoint(endpoint) for endpoint in self.s.endpoints.values(): if not endpoint.copro_ignore: if self.s.sdnc: if endpoint.copro_state == 'unknown': endpoint.p_next_copro_state = 'coprocessing' endpoint.queue() endpoint.p_prev_copro_states.append( (endpoint.copro_state, int(time.time()))) elif endpoint.copro_state in ['coprocessing']: cur_time = int(time.time()) # timeout after 2 times the reinvestigation frequency # in case something didn't report back, put back in an # unknown state if cur_time - endpoint.p_prev_copro_states[-1][ 1] > 2 * self.controller[ 'coprocessing_frequency']: self.logger.debug( 'timing out: {0} and setting to unknown'. format(endpoint.name)) self.s.unmirror_endpoint(endpoint) endpoint.unknown() endpoint.p_prev_copro_states.append( (endpoint.copro_state, int(time.time()))) else: if endpoint.state != 'nominal': endpoint.nominal() def process(self): global CTRL_C signal.signal(signal.SIGINT, partial(self.signal_handler)) while not CTRL_C['STOP']: time.sleep(1) found_work, item = self.get_q_item() if found_work: self.format_rabbit_message(item) self.schedule_mirroring() self.s.store_endpoints() def get_q_item(self): ''' attempt to get a work item from the queue m_queue -> (routing_key, body) a read from get_q_item should be of the form (boolean,(routing_key, body)) ''' found_work = False item = None global CTRL_C if not CTRL_C['STOP']: try: item = self.m_queue.get(False) found_work = True self.m_queue.task_done() except queue.Empty: # pragma: no cover pass return (found_work, item) def shutdown(self): ''' gracefully shut down. ''' self.s.clear_filters() for job in self.schedule.jobs: self.logger.debug('shutdown :{0}'.format(job)) self.schedule.cancel_job(job) if self.rabbit_channel_connection_local: self.rabbit_channel_connection_local.close() if self.rabbit_channel_connection_local_fa: self.rabbit_channel_connection_local_fa.close() self.logger.debug('SHUTTING DOWN') self.logger.debug('EXITING') sys.exit() def signal_handler(self, _signal, _frame): ''' hopefully eat a CTRL_C and signal system shutdown ''' global CTRL_C CTRL_C['STOP'] = True self.logger.debug('CTRL-C: {0}'.format(CTRL_C)) try: self.shutdown() except Exception as e: # pragma: no cover self.logger.debug( 'Failed to handle signal properly because: {0}'.format(str(e)))
class Monitor(object): def __init__(self, skip_rabbit): self.faucet_event = [] self.m_queue = queue.Queue() self.skip_rabbit = skip_rabbit self.logger = logger # get config options self.controller = Config().get_config() # timer class to call things periodically in own thread self.schedule = schedule # setup prometheus self.prom = Prometheus() try: self.prom.initialize_metrics() except Exception as e: # pragma: no cover self.logger.debug( 'Prometheus metrics are already initialized: {0}'.format( str(e))) Prometheus.start() # initialize sdnconnect self.s = SDNConnect() # retrieve endpoints from redis self.s.get_stored_endpoints() # set all retrieved endpoints to inactive at the start for endpoint in self.s.endpoints: if not endpoint.ignore: if endpoint.state != 'inactive': if endpoint.state == 'mirroring': endpoint.p_next_state = 'mirror' elif endpoint.state == 'reinvestigating': endpoint.p_next_state = 'reinvestigate' elif endpoint.state == 'queued': endpoint.p_next_state = 'queue' elif endpoint.state in ['known', 'abnormal']: endpoint.p_next_state = endpoint.state endpoint.endpoint_data['active'] = 0 endpoint.inactive() endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) # store changes to state self.s.store_endpoints() # schedule periodic scan of endpoints thread self.schedule.every(self.controller['scan_frequency']).seconds.do( partial(schedule_job_kickurl, func=self)) # schedule periodic reinvestigations thread self.schedule.every( self.controller['reinvestigation_frequency']).seconds.do( partial(schedule_job_reinvestigation, func=self)) # schedule all threads self.schedule_thread = threading.Thread(target=partial( schedule_thread_worker, schedule=self.schedule), name='st_worker') def format_rabbit_message(self, item): ''' read a message off the rabbit_q the message should be item = (routing_key,msg) ''' ret_val = {} routing_key, my_obj = item self.logger.debug('rabbit_message:{0}'.format(my_obj)) # my_obj: (hash,data) my_obj = json.loads(my_obj) self.logger.debug('routing_key:{0}'.format(routing_key)) if routing_key == 'poseidon.algos.decider': self.logger.debug('decider value:{0}'.format(my_obj)) # TODO if valid response then send along otherwise nothing for key in my_obj: ret_val[key] = my_obj[key] elif routing_key == self.controller['FA_RABBIT_ROUTING_KEY']: self.logger.debug('FAUCET Event:{0}'.format(my_obj)) for key in my_obj: ret_val[key] = my_obj[key] return ret_val def process(self): global CTRL_C signal.signal(signal.SIGINT, partial(self.signal_handler)) while not CTRL_C['STOP']: time.sleep(1) found_work, item = self.get_q_item() ml_returns = {} if found_work and item[0] == self.controller[ 'FA_RABBIT_ROUTING_KEY']: self.faucet_event.append(self.format_rabbit_message(item)) self.logger.debug('Faucet event: {0}'.format( self.faucet_event)) elif found_work: ml_returns = self.format_rabbit_message(item) self.logger.info('ML results: {0}'.format(ml_returns)) # process results from ml output and update impacted endpoints for ep in self.s.endpoints: if ep.name in ml_returns and 'valid' in ml_returns[ ep.name] and not ep.ignore: if ep.state in ['mirroring', 'reinvestigating']: status = Actions(ep, self.s.sdnc).unmirror_endpoint() if not status: self.logger.warning( 'Unable to unmirror the endpoint: {0}'. format(ep.name)) self.s.investigations -= 1 if ml_returns[ep.name]['valid']: ml_decision = None if 'decisions' in ml_returns[ ep.name] and 'behavior' in ml_returns[ ep.name]['decisions']: ml_decision = ml_returns[ ep.name]['decisions']['behavior'] if ml_decision == 'normal': ep.known() else: ep.abnormal() else: ep.unknown() ep.p_prev_states.append((ep.state, int(time.time()))) for endpoint in self.s.endpoints: if not endpoint.ignore: if endpoint.state == 'queued': if self.s.investigations < self.controller[ 'max_concurrent_reinvestigations']: self.s.investigations += 1 endpoint.trigger(endpoint.p_next_state) endpoint.p_next_state = None endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) status = Actions(endpoint, self.s.sdnc).mirror_endpoint() if not status: self.logger.warning( 'Unable to mirror the endpoint: {0}'. format(endpoint.name)) elif endpoint.state == 'unknown': # move to mirroring state if self.s.investigations < self.controller[ 'max_concurrent_reinvestigations']: self.s.investigations += 1 endpoint.mirror() endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) status = Actions(endpoint, self.s.sdnc).mirror_endpoint() if not status: self.logger.warning( 'Unable to mirror the endpoint: {0}'. format(endpoint.name)) else: endpoint.p_next_state = 'mirror' endpoint.queue() endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) elif endpoint.state in ['mirroring', 'reinvestigating']: cur_time = int(time.time()) # timeout after 2 times the reinvestigation frequency # in case something didn't report back, put back in an # unknown state if cur_time - endpoint.p_prev_states[-1][ 1] > 2 * self.controller[ 'reinvestigation_frequency']: status = Actions(endpoint, self.s.sdnc).unmirror_endpoint() if not status: self.logger.warning( 'Unable to unmirror the endpoint: {0}'. format(endpoint.name)) endpoint.unknown() self.s.investigations -= 1 endpoint.p_prev_states.append( (endpoint.state, int(time.time()))) def get_q_item(self): ''' attempt to get a work item from the queue m_queue -> (routing_key, body) a read from get_q_item should be of the form (boolean,(routing_key, body)) ''' found_work = False item = None global CTRL_C if not CTRL_C['STOP']: try: item = self.m_queue.get(False) found_work = True self.m_queue.task_done() except queue.Empty: # pragma: no cover pass return (found_work, item) def signal_handler(self, signal, frame): ''' hopefully eat a CTRL_C and signal system shutdown ''' global CTRL_C CTRL_C['STOP'] = True self.logger.debug('=================CTRLC{0}'.format(CTRL_C)) try: for job in self.schedule.jobs: self.logger.debug('CTRLC:{0}'.format(job)) self.schedule.cancel_job(job) self.rabbit_channel_connection_local.close() self.rabbit_channel_connection_local_fa.close() sys.exit() except BaseException: # pragma: no cover pass
def test_Prometheus(): """ Tests Prometheus """ p = Prometheus.get_metrics() hosts = [{ 'active': 0, 'source': 'poseidon', 'role': 'unknown', 'state': 'unknown', 'ipv4_os': 'unknown', 'tenant': 'vlan1', 'port': 1, 'segment': 'switch1', 'ipv4': '123.123.123.123', 'mac': '00:00:00:00:00:00', 'id': 'foo1', 'behavior': 1 }, { 'active': 1, 'source': 'poseidon', 'role': 'unknown', 'state': 'unknown', 'ipv4_os': 'unknown', 'tenant': 'vlan1', 'port': 1, 'segment': 'switch1', 'ipv4': '123.123.123.123', 'mac': '00:00:00:00:00:00', 'id': 'foo2', 'behavior': 1 }, { 'active': 0, 'source': 'poseidon', 'role': 'unknown', 'state': 'unknown', 'ipv4_os': 'unknown', 'tenant': 'vlan1', 'port': 1, 'segment': 'switch1', 'ipv4': '123.123.123.123', 'mac': '00:00:00:00:00:00', 'id': 'foo3', 'behavior': 1 }, { 'active': 1, 'source': 'poseidon1', 'role': 'unknown', 'state': 'unknown', 'ipv4_os': 'unknown', 'tenant': 'vlan1', 'port': 2, 'segment': 'switch1', 'ipv4': '2106::1', 'mac': '00:00:00:00:00:00', 'id': 'foo4', 'behavior': 1 }, { 'active': 1, 'source': 'poseidon', 'role': 'unknown', 'state': 'unknown', 'ipv4_os': 'unknown', 'tenant': 'vlan1', 'port': 1, 'segment': 'switch1', 'ipv4': '::', 'mac': '00:00:00:00:00:00', 'id': 'foo5', 'behavior': 1 }] p = Prometheus() p.update_metrics(hosts)