def _process_indicator(self, i): # searches if i.get('nolog', '0') in ['1', 1, True]: return if not i.get('itype'): i = Indicator( indicator=i['indicator'], tags='search', confidence=4, group='everyone', tlp='amber', ).__dict__() if not i.get('tags'): i['tags'] = [] if not i.get('confidence', 0): return if isinstance(i['confidence'], str): i['confidence'] = float(i['confidence']) if i['confidence'] < MIN_CONFIDENCE: return if 'predict' in i['tags']: return ii = Indicator(**i) if self._exclude(ii): return self._process_plugins(ii)
def test_firsttime_only(): l = arrow.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") i = Indicator('192.168.1.1', firsttime=l) assert i.firsttime == arrow.get(l).datetime s = str(i) i = json.loads(s) assert i.get('lasttime') is None
def start(self): router = Client(remote=self.router, token=self.token, nowait=True) plugins = self._load_plugins() socket = zmq.Context().socket(zmq.PULL) socket.SNDTIMEO = SNDTIMEO socket.set_hwm(ZMQ_HWM) logger.debug('connecting to {}'.format(self.hunters)) socket.connect(self.hunters) logger.debug('starting hunter') poller = zmq.Poller() poller.register(socket, zmq.POLLIN) while not self.exit.is_set(): try: s = dict(poller.poll(1000)) except SystemExit or KeyboardInterrupt: break if socket not in s: continue data = socket.recv_multipart() logger.debug(data) data = json.loads(data[0]) if isinstance(data, dict): if not data.get('indicator'): continue if not data.get('itype'): data = Indicator( indicator=data['indicator'], tags='search', confidence=10, group='everyone', tlp='amber', ).__dict__() if not data.get('tags'): data['tags'] = [] data = [data] for d in data: d = Indicator(**d) if d.indicator in ["", 'localhost', 'example.com']: continue if self.exclude.get(d.provider): for t in d.tags: if t in self.exclude[d.provider]: logger.debug('skipping: {}'.format(d.indicator)) for p in plugins: if p.is_advanced: if not HUNTER_ADVANCED: continue try: p.process(d, router) except Exception as e: logger.error(e) logger.error('[{}] giving up on: {}'.format(p, d))
def start(self): router = Client(remote=self.router, token=self.token, nowait=True, autoclose=False) plugins = self._load_plugins() socket = zmq.Context().socket(zmq.PULL) socket.SNDTIMEO = SNDTIMEO socket.set_hwm(ZMQ_HWM) logger.debug('connecting to {}'.format(self.hunters)) socket.connect(self.hunters) logger.debug('starting hunter') poller = zmq.Poller() poller.register(socket, zmq.POLLIN) while not self.exit.is_set(): try: s = dict(poller.poll(1000)) except SystemExit or KeyboardInterrupt: break if socket not in s: continue id, token, mtype, data = Msg().recv(socket) data = json.loads(data) if isinstance(data, dict): if not data.get('indicator'): continue if not data.get('itype'): try: data = Indicator( indicator=data['indicator'], tags='search', confidence=10, group='everyone', tlp='amber', ).__dict__() except InvalidIndicator: logger.debug('skipping invalid indicator: {}'.format( data['indicator'])) continue if not data.get('tags'): data['tags'] = [] data = [data] token = json.loads(token) for d in data: d = Indicator(**d) if d.confidence < HUNTER_MIN_CONFIDENCE: continue # prevent hunter recursion if disabled if not HUNTER_RECURSION and d.tags and 'hunter' in d.tags: continue if d.indicator in ["", 'localhost', 'example.com']: continue if self.exclude.get(d.provider): for t in d.tags: if t in self.exclude[d.provider]: logger.debug('skipping: {}'.format(d.indicator)) continue for p in plugins: if p.is_advanced: if not HUNTER_ADVANCED: continue try: p.process(i=d, router=router, user_token=token) except Exception as e: logger.error(e) logger.error('[{}] giving up on: {}'.format(p, d))
def start(self): router = Client(remote=self.router, token=self.token, nowait=True) plugins = self._load_plugins() socket = zmq.Context().socket(zmq.PULL) socket.SNDTIMEO = SNDTIMEO socket.set_hwm(ZMQ_HWM) logger.debug('connecting to {}'.format(self.hunters)) socket.connect(self.hunters) logger.debug('starting hunter') poller = zmq.Poller() poller.register(socket, zmq.POLLIN) while not self.exit.is_set(): try: s = dict(poller.poll(1000)) except SystemExit or KeyboardInterrupt: break if socket not in s: continue data = socket.recv_multipart() logger.debug(data) data = json.loads(data[0]) if isinstance(data, dict): if not data.get('indicator'): continue if not data.get('itype'): try: data = Indicator( indicator=data['indicator'], tags='search', confidence=10, group='everyone', tlp='amber', ).__dict__() except InvalidIndicator: logger.debug('skipping invalid indicator: {}'.format(data['indicator'])) continue if not data.get('tags'): data['tags'] = [] data = [data] for d in data: d = Indicator(**d) if d.indicator in ["", 'localhost', 'example.com']: continue if self.exclude.get(d.provider): for t in d.tags: if t in self.exclude[d.provider]: logger.debug('skipping: {}'.format(d.indicator)) continue for p in plugins: if p.is_advanced: if not HUNTER_ADVANCED: continue try: p.process(d, router) except Exception as e: logger.error(e) logger.error('[{}] giving up on: {}'.format(p, d))
def test_get_attr(): i = Indicator('128.205.1.1', tags='malware') assert i.get('tags')
def _process_message(self, message): data = message data = json.loads(data[0]) if not isinstance(data, dict): return if not data.get('indicator'): return if data['indicator'] in ["", 'localhost', 'example.com']: return # searches if not data.get('itype'): data = Indicator( indicator=data['indicator'], tags='search', confidence=4, group='everyone', tlp='amber', ).__dict__() if not data.get('tags'): data['tags'] = [] d = Indicator(**data) if self.exclude.get(d.provider): for t in d.tags: if t in self.exclude[d.provider]: logger.debug('skipping: {}'.format(d.indicator)) indicators = [] for p in self.plugins: try: indicators = p.process(d) indicators = [i.__dict__() for i in indicators] except KeyboardInterrupt: break except SystemExit: # sometimes cifsdk/zmq in will throw a sysexit if we sigint # and it's busy.. break except Exception as e: if 'SERVFAIL' in str(e): continue logger.error(e) logger.error('[{}] giving up on: {}'.format(p, d)) if logger.getEffectiveLevel() == logging.DEBUG: import traceback traceback.print_exc() continue try: self.router.indicators_create(indicators) except Exception as e: logger.error(e) if logger.getEffectiveLevel() == logging.DEBUG: import traceback traceback.print_exc() indicators = []
def start(self): plugins = load_plugins(cif.hunter.__path__) socket = zmq.Context().socket(zmq.PULL) socket.SNDTIMEO = SNDTIMEO socket.set_hwm(ZMQ_HWM) socket.setsockopt(zmq.LINGER, 3) socket.connect(HUNTER_ADDR) router = Client(remote=HUNTER_SINK_ADDR, token=self.token, nowait=True) poller = zmq.Poller() poller.register(socket, zmq.POLLIN) while not self.exit.is_set(): try: s = dict(poller.poll(1000)) except KeyboardInterrupt or SystemExit: break if socket not in s: continue data = socket.recv_multipart() data = json.loads(data[0]) logger.debug(data) if isinstance(data, dict): if not data.get('indicator'): continue # searches if not data.get('itype'): data = Indicator( indicator=data['indicator'], tags='search', confidence=4, group='everyone', tlp='amber', ).__dict__() if not data.get('tags'): data['tags'] = [] d = Indicator(**data) if d.indicator in ["", 'localhost', 'example.com']: continue if self.exclude.get(d.provider): for t in d.tags: if t in self.exclude[d.provider]: logger.debug('skipping: {}'.format(d.indicator)) for p in plugins: try: rv = p.process(d) if not rv: continue if not isinstance(rv, list): rv = [rv] rv = [i.__dict__() for i in rv] router.indicators_create(rv) except Exception as e: logger.error(e) logger.error('[{}] giving up on: {}'.format(p, d)) if logger.getEffectiveLevel() == logging.DEBUG: import traceback traceback.print_exc() socket.close() router.context.term() del router