def _process_indicator(self, i):
# searches
if i.get('nolog', '0') in ['1', 1, True]:
return
if not i.get('itype'):
i = Indicator(
indicator=i['indicator'],
tags='search',
confidence=4,
group='everyone',
tlp='amber',
).__dict__()
if not i.get('tags'):
i['tags'] = []
if not i.get('confidence', 0):
return
if isinstance(i['confidence'], str):
i['confidence'] = float(i['confidence'])
if i['confidence'] < MIN_CONFIDENCE:
return
if 'predict' in i['tags']:
return
ii = Indicator(**i)
if self._exclude(ii):
return
self._process_plugins(ii)
def test_firsttime_only():
l = arrow.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
i = Indicator('192.168.1.1', firsttime=l)
assert i.firsttime == arrow.get(l).datetime
s = str(i)
i = json.loads(s)
assert i.get('lasttime') is None
def start(self):
router = Client(remote=self.router, token=self.token, nowait=True)
plugins = self._load_plugins()
socket = zmq.Context().socket(zmq.PULL)
socket.SNDTIMEO = SNDTIMEO
socket.set_hwm(ZMQ_HWM)
logger.debug('connecting to {}'.format(self.hunters))
socket.connect(self.hunters)
logger.debug('starting hunter')
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
while not self.exit.is_set():
try:
s = dict(poller.poll(1000))
except SystemExit or KeyboardInterrupt:
break
if socket not in s:
continue
data = socket.recv_multipart()
logger.debug(data)
data = json.loads(data[0])
if isinstance(data, dict):
if not data.get('indicator'):
continue
if not data.get('itype'):
data = Indicator(
indicator=data['indicator'],
tags='search',
confidence=10,
group='everyone',
tlp='amber',
).__dict__()
if not data.get('tags'):
data['tags'] = []
data = [data]
for d in data:
d = Indicator(**d)
if d.indicator in ["", 'localhost', 'example.com']:
continue
if self.exclude.get(d.provider):
for t in d.tags:
if t in self.exclude[d.provider]:
logger.debug('skipping: {}'.format(d.indicator))
for p in plugins:
if p.is_advanced:
if not HUNTER_ADVANCED:
continue
try:
p.process(d, router)
except Exception as e:
logger.error(e)
logger.error('[{}] giving up on: {}'.format(p, d))
def start(self):
router = Client(remote=self.router,
token=self.token,
nowait=True,
autoclose=False)
plugins = self._load_plugins()
socket = zmq.Context().socket(zmq.PULL)
socket.SNDTIMEO = SNDTIMEO
socket.set_hwm(ZMQ_HWM)
logger.debug('connecting to {}'.format(self.hunters))
socket.connect(self.hunters)
logger.debug('starting hunter')
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
while not self.exit.is_set():
try:
s = dict(poller.poll(1000))
except SystemExit or KeyboardInterrupt:
break
if socket not in s:
continue
id, token, mtype, data = Msg().recv(socket)
data = json.loads(data)
if isinstance(data, dict):
if not data.get('indicator'):
continue
if not data.get('itype'):
try:
data = Indicator(
indicator=data['indicator'],
tags='search',
confidence=10,
group='everyone',
tlp='amber',
).__dict__()
except InvalidIndicator:
logger.debug('skipping invalid indicator: {}'.format(
data['indicator']))
continue
if not data.get('tags'):
data['tags'] = []
data = [data]
token = json.loads(token)
for d in data:
d = Indicator(**d)
if d.confidence < HUNTER_MIN_CONFIDENCE:
continue
# prevent hunter recursion if disabled
if not HUNTER_RECURSION and d.tags and 'hunter' in d.tags:
continue
if d.indicator in ["", 'localhost', 'example.com']:
continue
if self.exclude.get(d.provider):
for t in d.tags:
if t in self.exclude[d.provider]:
logger.debug('skipping: {}'.format(d.indicator))
continue
for p in plugins:
if p.is_advanced:
if not HUNTER_ADVANCED:
continue
try:
p.process(i=d, router=router, user_token=token)
except Exception as e:
logger.error(e)
logger.error('[{}] giving up on: {}'.format(p, d))
def start(self):
router = Client(remote=self.router, token=self.token, nowait=True)
plugins = self._load_plugins()
socket = zmq.Context().socket(zmq.PULL)
socket.SNDTIMEO = SNDTIMEO
socket.set_hwm(ZMQ_HWM)
logger.debug('connecting to {}'.format(self.hunters))
socket.connect(self.hunters)
logger.debug('starting hunter')
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
while not self.exit.is_set():
try:
s = dict(poller.poll(1000))
except SystemExit or KeyboardInterrupt:
break
if socket not in s:
continue
data = socket.recv_multipart()
logger.debug(data)
data = json.loads(data[0])
if isinstance(data, dict):
if not data.get('indicator'):
continue
if not data.get('itype'):
try:
data = Indicator(
indicator=data['indicator'],
tags='search',
confidence=10,
group='everyone',
tlp='amber',
).__dict__()
except InvalidIndicator:
logger.debug('skipping invalid indicator: {}'.format(data['indicator']))
continue
if not data.get('tags'):
data['tags'] = []
data = [data]
for d in data:
d = Indicator(**d)
if d.indicator in ["", 'localhost', 'example.com']:
continue
if self.exclude.get(d.provider):
for t in d.tags:
if t in self.exclude[d.provider]:
logger.debug('skipping: {}'.format(d.indicator))
continue
for p in plugins:
if p.is_advanced:
if not HUNTER_ADVANCED:
continue
try:
p.process(d, router)
except Exception as e:
logger.error(e)
logger.error('[{}] giving up on: {}'.format(p, d))
def test_get_attr():
i = Indicator('128.205.1.1', tags='malware')
assert i.get('tags')
def _process_message(self, message):
data = message
data = json.loads(data[0])
if not isinstance(data, dict):
return
if not data.get('indicator'):
return
if data['indicator'] in ["", 'localhost', 'example.com']:
return
# searches
if not data.get('itype'):
data = Indicator(
indicator=data['indicator'],
tags='search',
confidence=4,
group='everyone',
tlp='amber',
).__dict__()
if not data.get('tags'):
data['tags'] = []
d = Indicator(**data)
if self.exclude.get(d.provider):
for t in d.tags:
if t in self.exclude[d.provider]:
logger.debug('skipping: {}'.format(d.indicator))
indicators = []
for p in self.plugins:
try:
indicators = p.process(d)
indicators = [i.__dict__() for i in indicators]
except KeyboardInterrupt:
break
except SystemExit:
# sometimes cifsdk/zmq in will throw a sysexit if we sigint
# and it's busy..
break
except Exception as e:
if 'SERVFAIL' in str(e):
continue
logger.error(e)
logger.error('[{}] giving up on: {}'.format(p, d))
if logger.getEffectiveLevel() == logging.DEBUG:
import traceback
traceback.print_exc()
continue
try:
self.router.indicators_create(indicators)
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
import traceback
traceback.print_exc()
indicators = []
def start(self):
plugins = load_plugins(cif.hunter.__path__)
socket = zmq.Context().socket(zmq.PULL)
socket.SNDTIMEO = SNDTIMEO
socket.set_hwm(ZMQ_HWM)
socket.setsockopt(zmq.LINGER, 3)
socket.connect(HUNTER_ADDR)
router = Client(remote=HUNTER_SINK_ADDR, token=self.token, nowait=True)
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
while not self.exit.is_set():
try:
s = dict(poller.poll(1000))
except KeyboardInterrupt or SystemExit:
break
if socket not in s:
continue
data = socket.recv_multipart()
data = json.loads(data[0])
logger.debug(data)
if isinstance(data, dict):
if not data.get('indicator'):
continue
# searches
if not data.get('itype'):
data = Indicator(
indicator=data['indicator'],
tags='search',
confidence=4,
group='everyone',
tlp='amber',
).__dict__()
if not data.get('tags'):
data['tags'] = []
d = Indicator(**data)
if d.indicator in ["", 'localhost', 'example.com']:
continue
if self.exclude.get(d.provider):
for t in d.tags:
if t in self.exclude[d.provider]:
logger.debug('skipping: {}'.format(d.indicator))
for p in plugins:
try:
rv = p.process(d)
if not rv:
continue
if not isinstance(rv, list):
rv = [rv]
rv = [i.__dict__() for i in rv]
router.indicators_create(rv)
except Exception as e:
logger.error(e)
logger.error('[{}] giving up on: {}'.format(p, d))
if logger.getEffectiveLevel() == logging.DEBUG:
import traceback
traceback.print_exc()
socket.close()
router.context.term()
del router
