def __init__(self, pull=GATHERER_ADDR, push=GATHERER_SINK_ADDR): multiprocessing.Process.__init__(self) self.pull = pull self.push = push self.exit = multiprocessing.Event() self.gatherers = load_plugins(cif.gatherer.__path__)
def __init__(self, **kwargs): Actor.__init__(self, **kwargs) self.plugins_ext = init_plugins(PLUGINS_NAMESPACE) self.plugins = load_plugins(enrichers.__path__) + \ [self.plugins_ext[p] for p in self.plugins_ext] self.push_s = None
def __init__(self, token=TOKEN): MyProcess.__init__(self) self.token = token self.exclude = {} self.settings = settings(CONFIG_PATH) self._init_token() self._init_exclude() self.plugins = load_plugins(cif.hunter.__path__) self.router = None
def test_hunter_plugins(): plugins = load_plugins(cif.hunter.__path__) count = 0 indicators = [] for d in range(0, 1): i = Indicator(indicator=fake.domain_name(), tags=['malware']) indicators.append(i) indicators.append(Indicator('csirtgadgets.com', tags=['botnet'])) indicators.append(Indicator('gfycat.com', tags=['exploit'])) indicators.append(Indicator('http://csirtgadgets.com', tags=['botnet'])) for p in plugins: rv = p.process(next(i for i in indicators)) rv = list(r for r in rv) if not rv or len(rv) == 0: continue rv = [i.__dict__() for i in rv] count += len(rv)
class Hunter(Actor): plugins = load_plugins(hunters.__path__) def __init__(self): Actor.__init__(self) self.exclude = {} self.router = None self._init_exclude() self.plugins_ext = init_plugins('csirtg_hunters_') for p in self.plugins_ext: self.plugins.append(self.plugins_ext[p]) def _init_exclude(self): if not EXCLUDE: return for e in EXCLUDE.split(','): provider, tag = e.split(':') if not self.exclude.get(provider): self.exclude[provider] = set() logger.debug('setting hunter to skip: {}/{}'.format(provider, tag)) self.exclude[provider].add(tag) def _exclude(self, d): if d.indicator in EXCLUSIONS: return True if not self.exclude.get(d.provider): return for t in d.tags: if t in self.exclude[d.provider]: logger.debug('skipping: {}'.format(d.indicator)) return True @staticmethod def _process_plugin(p, i): try: indicators = p.process(i) if not indicators: return return [ii.__dict__() for ii in indicators if ii] except (KeyboardInterrupt, SystemExit): return except Exception as e: if 'SERVFAIL' not in str(e): logger.error(e, exc_info=True) logger.error(f"[{p}] giving up on {i}") def _process_plugins(self, i): for p in self.plugins: try: indicators = self._process_plugin(p, i) except Exception as e: logger.error(e, exc_info=True) continue if not indicators or len(indicators) == 0: continue for ii in indicators: ii['iid'] = i.uuid try: self.router.indicators_create(indicators, fireball=False) except zmq.error.Again: logger.error('EAGAIN: unable to create indicators.') logger.debug(len(indicators)) logger.debug(indicators) except Exception as e: # TODO- catch low HWM (EAGAIN) and backoff... logger.error(e, exc_info=True) def _process_indicator(self, i): # searches if i.get('nolog', '0') in ['1', 1, True]: return if not i.get('itype'): i = Indicator( indicator=i['indicator'], tags='search', confidence=4, group='everyone', tlp='amber', ).__dict__() if not i.get('tags'): i['tags'] = [] if not i.get('confidence', 0): return if isinstance(i['confidence'], str): i['confidence'] = float(i['confidence']) if i['confidence'] < MIN_CONFIDENCE: return if 'predict' in i['tags']: return ii = Indicator(**i) if self._exclude(ii): return self._process_plugins(ii) def _process_message(self, message): try: [ self._process_indicator(i) for i in Msg().from_frame(message).data ] except (KeyboardInterrupt, SystemExit): return except Exception as e: logger.error(e) if logger.getEffectiveLevel() == logging.DEBUG: traceback.print_exc() def start(self): loop = IOLoop() s = Context().socket(zmq.PULL) s.set_hwm(ZMQ_HWM) socket = zmqstream.ZMQStream(s, loop) socket.on_recv(self._process_message) logger.debug(f"connecting to: {HUNTER_ADDR}") socket.connect(HUNTER_ADDR) # this needs to be done here self.router = Client(remote=HUNTER_SINK_ADDR, nowait=True, autoclose=False) try: loop.start() except KeyboardInterrupt as e: loop.stop() for ss in [s, self.router.socket]: ss.close() self.stop()
def __init__(self, pull=GATHERER_ADDR, push=GATHERER_SINK_ADDR, **kwargs): MyProcess.__init__(self) self.pull = pull self.push = push self.gatherers = load_plugins(cif.gatherer.__path__)
def start(self): plugins = load_plugins(cif.hunter.__path__) socket = zmq.Context().socket(zmq.PULL) socket.SNDTIMEO = SNDTIMEO socket.set_hwm(ZMQ_HWM) socket.setsockopt(zmq.LINGER, 3) socket.connect(HUNTER_ADDR) router = Client(remote=HUNTER_SINK_ADDR, token=self.token, nowait=True) poller = zmq.Poller() poller.register(socket, zmq.POLLIN) while not self.exit.is_set(): try: s = dict(poller.poll(1000)) except KeyboardInterrupt or SystemExit: break if socket not in s: continue data = socket.recv_multipart() data = json.loads(data[0]) logger.debug(data) if isinstance(data, dict): if not data.get('indicator'): continue # searches if not data.get('itype'): data = Indicator( indicator=data['indicator'], tags='search', confidence=4, group='everyone', tlp='amber', ).__dict__() if not data.get('tags'): data['tags'] = [] d = Indicator(**data) if d.indicator in ["", 'localhost', 'example.com']: continue if self.exclude.get(d.provider): for t in d.tags: if t in self.exclude[d.provider]: logger.debug('skipping: {}'.format(d.indicator)) for p in plugins: try: rv = p.process(d) if not rv: continue if not isinstance(rv, list): rv = [rv] rv = [i.__dict__() for i in rv] router.indicators_create(rv) except Exception as e: logger.error(e) logger.error('[{}] giving up on: {}'.format(p, d)) if logger.getEffectiveLevel() == logging.DEBUG: import traceback traceback.print_exc() socket.close() router.context.term() del router