def __init__(self, pull=GATHERER_ADDR, push=GATHERER_SINK_ADDR):
multiprocessing.Process.__init__(self)
self.pull = pull
self.push = push
self.exit = multiprocessing.Event()
self.gatherers = load_plugins(cif.gatherer.__path__)
def __init__(self, **kwargs):
Actor.__init__(self, **kwargs)
self.plugins_ext = init_plugins(PLUGINS_NAMESPACE)
self.plugins = load_plugins(enrichers.__path__) + \
[self.plugins_ext[p] for p in self.plugins_ext]
self.push_s = None
def __init__(self, token=TOKEN):
MyProcess.__init__(self)
self.token = token
self.exclude = {}
self.settings = settings(CONFIG_PATH)
self._init_token()
self._init_exclude()
self.plugins = load_plugins(cif.hunter.__path__)
self.router = None
def test_hunter_plugins():
plugins = load_plugins(cif.hunter.__path__)
count = 0
indicators = []
for d in range(0, 1):
i = Indicator(indicator=fake.domain_name(), tags=['malware'])
indicators.append(i)
indicators.append(Indicator('csirtgadgets.com', tags=['botnet']))
indicators.append(Indicator('gfycat.com', tags=['exploit']))
indicators.append(Indicator('http://csirtgadgets.com', tags=['botnet']))
for p in plugins:
rv = p.process(next(i for i in indicators))
rv = list(r for r in rv)
if not rv or len(rv) == 0:
continue
rv = [i.__dict__() for i in rv]
count += len(rv)
class Hunter(Actor):
plugins = load_plugins(hunters.__path__)
def __init__(self):
Actor.__init__(self)
self.exclude = {}
self.router = None
self._init_exclude()
self.plugins_ext = init_plugins('csirtg_hunters_')
for p in self.plugins_ext:
self.plugins.append(self.plugins_ext[p])
def _init_exclude(self):
if not EXCLUDE:
return
for e in EXCLUDE.split(','):
provider, tag = e.split(':')
if not self.exclude.get(provider):
self.exclude[provider] = set()
logger.debug('setting hunter to skip: {}/{}'.format(provider, tag))
self.exclude[provider].add(tag)
def _exclude(self, d):
if d.indicator in EXCLUSIONS:
return True
if not self.exclude.get(d.provider):
return
for t in d.tags:
if t in self.exclude[d.provider]:
logger.debug('skipping: {}'.format(d.indicator))
return True
@staticmethod
def _process_plugin(p, i):
try:
indicators = p.process(i)
if not indicators:
return
return [ii.__dict__() for ii in indicators if ii]
except (KeyboardInterrupt, SystemExit):
return
except Exception as e:
if 'SERVFAIL' not in str(e):
logger.error(e, exc_info=True)
logger.error(f"[{p}] giving up on {i}")
def _process_plugins(self, i):
for p in self.plugins:
try:
indicators = self._process_plugin(p, i)
except Exception as e:
logger.error(e, exc_info=True)
continue
if not indicators or len(indicators) == 0:
continue
for ii in indicators:
ii['iid'] = i.uuid
try:
self.router.indicators_create(indicators, fireball=False)
except zmq.error.Again:
logger.error('EAGAIN: unable to create indicators.')
logger.debug(len(indicators))
logger.debug(indicators)
except Exception as e:
# TODO- catch low HWM (EAGAIN) and backoff...
logger.error(e, exc_info=True)
def _process_indicator(self, i):
# searches
if i.get('nolog', '0') in ['1', 1, True]:
return
if not i.get('itype'):
i = Indicator(
indicator=i['indicator'],
tags='search',
confidence=4,
group='everyone',
tlp='amber',
).__dict__()
if not i.get('tags'):
i['tags'] = []
if not i.get('confidence', 0):
return
if isinstance(i['confidence'], str):
i['confidence'] = float(i['confidence'])
if i['confidence'] < MIN_CONFIDENCE:
return
if 'predict' in i['tags']:
return
ii = Indicator(**i)
if self._exclude(ii):
return
self._process_plugins(ii)
def _process_message(self, message):
try:
[
self._process_indicator(i)
for i in Msg().from_frame(message).data
]
except (KeyboardInterrupt, SystemExit):
return
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
traceback.print_exc()
def start(self):
loop = IOLoop()
s = Context().socket(zmq.PULL)
s.set_hwm(ZMQ_HWM)
socket = zmqstream.ZMQStream(s, loop)
socket.on_recv(self._process_message)
logger.debug(f"connecting to: {HUNTER_ADDR}")
socket.connect(HUNTER_ADDR)
# this needs to be done here
self.router = Client(remote=HUNTER_SINK_ADDR,
nowait=True,
autoclose=False)
try:
loop.start()
except KeyboardInterrupt as e:
loop.stop()
for ss in [s, self.router.socket]:
ss.close()
self.stop()
def __init__(self, pull=GATHERER_ADDR, push=GATHERER_SINK_ADDR, **kwargs):
MyProcess.__init__(self)
self.pull = pull
self.push = push
self.gatherers = load_plugins(cif.gatherer.__path__)
def start(self):
plugins = load_plugins(cif.hunter.__path__)
socket = zmq.Context().socket(zmq.PULL)
socket.SNDTIMEO = SNDTIMEO
socket.set_hwm(ZMQ_HWM)
socket.setsockopt(zmq.LINGER, 3)
socket.connect(HUNTER_ADDR)
router = Client(remote=HUNTER_SINK_ADDR, token=self.token, nowait=True)
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
while not self.exit.is_set():
try:
s = dict(poller.poll(1000))
except KeyboardInterrupt or SystemExit:
break
if socket not in s:
continue
data = socket.recv_multipart()
data = json.loads(data[0])
logger.debug(data)
if isinstance(data, dict):
if not data.get('indicator'):
continue
# searches
if not data.get('itype'):
data = Indicator(
indicator=data['indicator'],
tags='search',
confidence=4,
group='everyone',
tlp='amber',
).__dict__()
if not data.get('tags'):
data['tags'] = []
d = Indicator(**data)
if d.indicator in ["", 'localhost', 'example.com']:
continue
if self.exclude.get(d.provider):
for t in d.tags:
if t in self.exclude[d.provider]:
logger.debug('skipping: {}'.format(d.indicator))
for p in plugins:
try:
rv = p.process(d)
if not rv:
continue
if not isinstance(rv, list):
rv = [rv]
rv = [i.__dict__() for i in rv]
router.indicators_create(rv)
except Exception as e:
logger.error(e)
logger.error('[{}] giving up on: {}'.format(p, d))
if logger.getEffectiveLevel() == logging.DEBUG:
import traceback
traceback.print_exc()
socket.close()
router.context.term()
del router