def _search_bulk(filters):
try:
with Client(ROUTER_ADDR, session['token']) as client:
r = client.indicators_search(filters)
except InvalidSearch as e:
return api.abort(400)
except AuthError as e:
return api.abort(401)
except zmq.error.Again as e:
return api.abort(503)
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
traceback.print_exc()
if 'invalid search' in str(e):
return api.abort(400, str(e))
return api.abort(500)
return r
def start(self):
loop = ioloop.IOLoop()
s = zmq.Context().socket(zmq.PULL)
s.SNDTIMEO = SNDTIMEO
s.set_hwm(ZMQ_HWM)
s.setsockopt(zmq.LINGER, 3)
socket = zmqstream.ZMQStream(s, loop)
socket.on_recv(self._process_message)
socket.connect(HUNTER_ADDR)
# this needs to be done here
self.router = Client(remote=HUNTER_SINK_ADDR,
token=self.token,
nowait=True,
autoclose=False)
try:
loop.start()
except KeyboardInterrupt as e:
loop.stop()
self.router.socket.close()
def _ping(self, write=False):
try:
if write:
with Client(ROUTER_ADDR, session['token']) as cli:
r = cli.ping_write()
else:
with Client(ROUTER_ADDR, session['token']) as cli:
r = cli.ping()
except TimeoutError:
return api.abort(408)
except AuthError:
return api.abort(401)
if not r:
return api.abort(503)
return {'status': 'success', 'data': time.time()}
def post(self):
"""Create an Indicator"""
if len(request.data) == 0:
return 'missing indicator', 422
fireball = False
nowait = request.args.get('nowait', False)
if request.headers.get('Content-Length'):
logger.debug('size: %0.2f kb' %
(float(request.headers['Content-Length']) / 1024))
if int(request.headers['Content-Length']) > 5000:
fireball = True
try:
with Client(ROUTER_ADDR, session['token']) as cli:
r = cli.indicators_create(request.data,
nowait=nowait,
fireball=fireball)
if nowait:
r = {'message': 'pending'}
else:
count = 0
if isinstance(r, list):
for rr in r:
count += rr.get('data', 0)
r = count
except SubmissionFailed as e:
logger.error(e)
return api.abort(422)
except TimeoutError as e:
return api.abort(408)
except CIFBusy:
return api.abort(503)
except AuthError:
return api.abort(401)
except Exception as e:
logger.error(e)
import traceback
traceback.print_exc()
return api.abort(400)
return {'data': r, 'message': 'success'}, 201
def _pull(self, filters={}):
try:
r = Client(ROUTER_ADDR, session['token']).stats_search(filters)
except InvalidSearch as e:
return api.abort(400)
except AuthError as e:
return api.abort(401)
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
traceback.print_exc()
return api.abort(500)
return r
def get(self):
"""List Graph Nodes"""
try:
r = Client(ROUTER_ADDR, session['token']).graph_search({})
except InvalidSearch as e:
return api.abort(400)
except AuthError as e:
return api.abort(401)
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
traceback.print_exc()
return api.abort(500)
return {'status': 'success', 'data': r}, 200
def __init_page_output(self, page_num):
output_file = open('/home/cif/palo_paged_indicators.txt', 'w')
filters = {
'tags': 'botnet,phishing,malware,scanner,bruteforce,darknet',
'itype': 'ipv4',
'limit': '150000'
}
with Client(ROUTER_ADDR, os.getenv('CIF_TOKEN')) as client:
results = client.indicators_search(filters)
all_indicators_dirty = []
all_indicators_clean = []
# all_indicators_dirty is a list of dictionaries containing only the "id" and "indicator" attributes
# the items within the all_indicators_dirty list are sorted by the dictionary item's "id" attribute
for item in results:
my_dict = {}
my_dict['id'] = item.get('id')
my_dict['indicator'] = item.get('indicator')
all_indicators_dirty.append(my_dict)
all_indicators_dirty.sort(key=lambda x: x["id"])
# the "all_indicators_clean" list is the entire list of indicators
for obj in all_indicators_dirty:
all_indicators_clean.append(obj["indicator"])
length_of_indicators = len(all_indicators_clean)
# each page can contain no more than 5000 indicators
# initialize index count based on paging
index_count = (int(page_num) * 5000) - 5000
for num in range(5000):
if (index_count > length_of_indicators - 1):
break
else:
output_file.write(all_indicators_clean[index_count])
output_file.write("\n")
index_count += 1
output_file.close()
return
def firehose(ws):
"""Firehose"""
from cifsdk.client.zmq import ZMQ as Client
from cifsdk.constants import ROUTER_ADDR
from cifsdk.exceptions import AuthError, TimeoutError
# check authorization
t = pull_token()
try:
r = Client(ROUTER_ADDR, t).tokens_search(filters={'q': t})
except TimeoutError:
return api.abort(408)
except AuthError:
return api.abort(401)
if not r:
return api.abort(503)
ctx = zmq.Context()
router = ctx.socket(zmq.SUB)
router.connect(STREAM_ADDR)
poller = zmq.Poller()
poller.register(router, zmq.POLLIN)
while not ws.closed:
try:
s = dict(poller.poll(1000))
except KeyboardInterrupt or SystemExit:
break
if router not in s:
continue
message = router.recv_multipart()
ws.send(message[0])
def test_zmq():
cli = Client(ROUTER_ADDR, '12345')
assert cli.remote == ROUTER_ADDR
assert cli.token == '12345'
def firehose(ws):
"""Firehose"""
t = pull_token()
# check authorization
try:
r = Client(ROUTER_ADDR, t).tokens_search(filters={'q': t})
if not r:
return api.abort(401)
except TimeoutError:
return api.abort(408)
except AuthError:
return api.abort(401)
except Exception as e:
import traceback
traceback.print_exc()
return api.abort(503)
ctx = zmq.Context()
router = ctx.socket(zmq.SUB)
logger.debug('connecting: %s' % STREAM_ADDR)
router.connect(STREAM_ADDR)
router.setsockopt(zmq.SUBSCRIBE, b'')
poller = zmq.Poller()
poller.register(router, zmq.POLLIN)
ws.send("connected")
while not ws.closed:
try:
s = dict(poller.poll(5000))
except KeyboardInterrupt or SystemExit:
break
if ws.closed:
break
if len(s) == 0:
try:
ws.send('ping')
except Exception as e:
break
m = ws.receive()
if m != 'pong':
break
if router not in s:
continue
message = router.recv_multipart()
ws.send(message[0])
logger.debug('cleaning up ws client..')
router.close()
del router
def start(self):
plugins = load_plugins(cif.hunter.__path__)
socket = zmq.Context().socket(zmq.PULL)
socket.SNDTIMEO = SNDTIMEO
socket.set_hwm(ZMQ_HWM)
socket.setsockopt(zmq.LINGER, 3)
socket.connect(HUNTER_ADDR)
router = Client(remote=HUNTER_SINK_ADDR, token=self.token, nowait=True)
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
while not self.exit.is_set():
try:
s = dict(poller.poll(1000))
except KeyboardInterrupt or SystemExit:
break
if socket not in s:
continue
data = socket.recv_multipart()
data = json.loads(data[0])
logger.debug(data)
if isinstance(data, dict):
if not data.get('indicator'):
continue
# searches
if not data.get('itype'):
data = Indicator(
indicator=data['indicator'],
tags='search',
confidence=4,
group='everyone',
tlp='amber',
).__dict__()
if not data.get('tags'):
data['tags'] = []
d = Indicator(**data)
if d.indicator in ["", 'localhost', 'example.com']:
continue
if self.exclude.get(d.provider):
for t in d.tags:
if t in self.exclude[d.provider]:
logger.debug('skipping: {}'.format(d.indicator))
for p in plugins:
try:
rv = p.process(d)
if not rv:
continue
if not isinstance(rv, list):
rv = [rv]
rv = [i.__dict__() for i in rv]
router.indicators_create(rv)
except Exception as e:
logger.error(e)
logger.error('[{}] giving up on: {}'.format(p, d))
if logger.getEffectiveLevel() == logging.DEBUG:
import traceback
traceback.print_exc()
socket.close()
router.context.term()
del router