def __init__(self,
context,
remote=HUNTER_ADDR,
router=ROUTER_ADDR,
token=None):
self.logger = logging.getLogger(__name__)
self.context = context
self.socket = self.context.socket(zmq.PULL)
self.plugins = self._load_plugins()
self.hunters = remote
# TODO - convert this to an async socket
self.router = Client(remote=router, token=token)
def patch(self):
cli = Client(remote, pull_token())
try:
r = cli.tokens_edit(request.data)
except AuthError:
return jsonify_unauth()
except Exception as e:
logger.error(e)
return jsonify_unknown()
if not r:
return jsonify_unauth()
return jsonify_success(r)
def post(self):
fireball = False
nowait = request.args.get('nowait', False)
if request.headers.get('Content-Length'):
logger.debug('content-length: %s' %
request.headers['Content-Length'])
if int(request.headers['Content-Length']) > 5000:
logger.info('fireball mode')
fireball = True
try:
r = Client(remote,
pull_token()).indicators_create(request.data,
nowait=nowait,
fireball=fireball)
if nowait:
r = 'pending'
except SubmissionFailed as e:
logger.error(e)
return jsonify_unknown(msg='submission failed: %s' % e, code=422)
except RuntimeError as e:
logger.error(e)
return jsonify_unknown(
msg='submission failed, check logs for more information',
code=422)
except TimeoutError as e:
logger.error(e)
return jsonify_unknown(
'submission failed, check logs for more information', 408)
except CIFBusy:
return jsonify_busy()
except Exception as e:
logger.error(e)
return jsonify_unknown(
'submission failed, check logs for more information', 422)
except AuthError:
return jsonify_unauth()
return jsonify_success(r, code=201)
def delete(self, token_id):
if not session['admin']:
return redirect('/u/login', code=401)
filters = {}
if token_id:
filters['token'] = token_id
filters['username'] = None
filters = json.dumps(filters)
try:
r = Client(remote, HTTPD_TOKEN).tokens_delete(filters)
except Exception as e:
logger.error(e)
flash(e, 'error')
else:
flash('success', 'success')
return redirect(url_for('/u/tokens'))
def _search_bulk(f):
try:
with Client() as client:
r = client.indicators_search(f)
except zmq.error.Again as e:
return api.abort(503)
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
traceback.print_exc()
if 'invalid search' in str(e):
logger.error(e)
return api.abort(400, str(e))
return api.abort(500)
return r
def get(self):
cli = Client(remote, pull_token())
filters = {}
for f in TOKEN_FILTERS:
filters[f] = request.args.get(f)
if current_app.config.get('dummy'):
return jsonify_success()
try:
r = cli.tokens_search(filters)
except AuthError:
return jsonify_unauth()
except Exception as e:
logger.error(e)
return jsonify_unknown()
return jsonify_success(r)
def get(self):
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if request.args.get('q'):
filters['indicator'] = request.args.get('q')
if current_app.config.get('dummy'):
r = DummyClient(remote, pull_token()).indicators_search(filters)
return jsonify_success(r)
try:
r = Client(remote, pull_token()).indicators_search(filters,
decode=False)
except RuntimeError as e:
logger.error(e)
return jsonify_unknown(msg='search failed')
except InvalidSearch as e:
return jsonify_unknown(msg='invalid search', code=400)
except AuthError:
return jsonify_unauth()
except Exception as e:
logger.error(e)
return jsonify_unknown(
msg='search failed, system may be too busy, check back later')
response = current_app.response_class(r, mimetype='application/json')
if isinstance(r, basestring):
if '"message":"unauthorized"' in r and '"message":"unauthorized"' in r:
response.status_code = 401
return response
return response
def search():
"""
Search controller
:param str q: query term (ex: example.org, 1.2.3.4, 1.2.3.0/24)
:param dict filters: query filters
:param int limit: limit search results (reporttime desc)
:return: { 'message': 'success', 'data': [] }
"""
remote = ROUTER_ADDR
if app.config.get('CIF_ROUTER_ADDR'):
remote = app.config['CIF_ROUTER_ADDR']
filters = {}
for k in VALID_FILTERS:
if request.args.get(k):
filters[k] = request.args.get(k)
if request.args.get('q'):
filters['indicator'] = request.args.get('q')
try:
if app.config.get('dummy'):
r = DummyClient(remote, pull_token()).indicators_search(filters)
else:
r = Client(remote, pull_token()).indicators_search(filters)
response = jsonify({"message": "success", "data": r})
response.status_code = 200
except AuthError as e:
response = jsonify({
'message': 'unauthorized',
'data': [],
'status': 'failed'
})
response.status_code = 401
return response
def delete(self):
try:
data = request.data.decode('utf-8')
with Client(remote, pull_token()) as cli:
r = cli.indicators_delete(data)
except RuntimeError as e:
logger.error(e)
return jsonify_unknown(msg='submission failed, check logs for more information', code=422)
except TimeoutError as e:
logger.error(e)
return jsonify_unknown('submission failed, check logs for more information', 408)
except Exception as e:
logger.error(e)
return jsonify_unknown('submission failed, check logs for more information', 422)
except AuthError:
return jsonify_unauth()
return jsonify_success(r)
def post(self):
cli = Client(remote, pull_token())
if not request.data:
return jsonify_unknown('missing data', 400)
if current_app.config.get('dummy'):
return jsonify_success()
try:
r = cli.tokens_create(request.data)
except AuthError:
return jsonify_unauth()
except Exception as e:
logger.error(e)
return jsonify_unknown()
if not r:
return jsonify_unknown('create failed', 400)
return jsonify_success(r, code=201)
def login():
if request.method == 'GET':
return render_template('login.html')
if request.method == 'POST':
from cifsdk.client.zeromq import ZMQ as Client
if request.form['token'] == '':
return render_template('login.html')
c = Client(remote, HTTPD_TOKEN)
rv = c.tokens_search({'token': request.form['token']})
if len(rv) == 0:
return render_template('login.html', code=401)
user = rv[0]
if user['revoked']:
return render_template('login.html', code=401)
for e in ['username', 'token', 'admin', 'read', 'write', 'groups']:
session[e] = user[e]
return redirect(url_for('/u/search'))
def get(self):
remote = ROUTER_ADDR
if current_app.config.get('CIF_ROUTER_ADDR'):
remote = current_app.config['CIF_ROUTER_ADDR']
write = request.args.get('write', None)
if current_app.config.get('dummy'):
r = DummyClient(remote, pull_token()).ping(write=write)
return jsonify_success(r)
try:
r = Client(remote, pull_token()).ping(write=write)
except TimeoutError:
return jsonify_unknown(msg='timeout', code=408)
except AuthError:
return jsonify_unauth()
if not r:
return jsonify_unknown(503)
return jsonify_success(r)
def start(self):
# TODO - convert this to an async socket
router = Client(remote=self.router, token=self.token, nowait=True)
plugins = self._load_plugins()
socket = zmq.Context().socket(zmq.PULL)
socket.SNDTIMEO = SNDTIMEO
socket.set_hwm(ZMQ_HWM)
logger.debug('connecting to {}'.format(self.hunters))
socket.connect(self.hunters)
logger.debug('starting hunter')
try:
while True:
logger.debug('waiting...')
data = socket.recv()
logger.debug(data)
data = json.loads(data)
if isinstance(data, dict):
data = [data]
for d in data:
d = Indicator(**d)
for p in plugins:
try:
p.process(d, router)
except Exception as e:
logger.error(e)
traceback.print_exc()
logger.error('giving up on: {}'.format(d))
except KeyboardInterrupt:
logger.info('shutting down hunter...')
return
def DataTables():
filters = {}
if session['filters'].get('q'):
q = session['filters'].get('q')
if q in ['ipv4', 'ipv6', 'fqdn', 'url', 'email']:
filters['itype'] = q
else:
filters['indicator'] = q
if session['filters'].get('confidence'):
filters['confidence'] = session['filters'].get('confidence')
if session['filters'].get('provider'):
filters['provider'] = session['filters'].get('provider')
if session['filters'].get('group'):
filters['groups'] = session['filters'].get('group')
if session['filters'].get('tags'):
filters['tags'] = session['filters'].get('tags')
if session['filters'].get('reporttime'):
filters['reporttime'] = session['filters'].get('reporttime')
if not session['filters']:
return []
try:
r = Client(remote, session['token']).indicators_search(filters)
except Exception as e:
logger.error(e)
flash(e, 'error')
response = []
else:
response = r
finally:
session['filters'] = {}
return response
def start(self):
router = Client(remote=self.router, token=self.token, nowait=True)
plugins = self._load_plugins()
socket = zmq.Context().socket(zmq.PULL)
socket.SNDTIMEO = SNDTIMEO
socket.set_hwm(ZMQ_HWM)
logger.debug('connecting to {}'.format(self.hunters))
socket.connect(self.hunters)
logger.debug('starting hunter')
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
while not self.exit.is_set():
try:
s = dict(poller.poll(1000))
except SystemExit or KeyboardInterrupt:
break
if socket not in s:
continue
data = socket.recv_multipart()
logger.debug(data)
data = json.loads(data[0])
if isinstance(data, dict):
if not data.get('indicator'):
continue
if not data.get('itype'):
data = Indicator(
indicator=data['indicator'],
tags='search',
confidence=10,
group='everyone',
tlp='amber',
).__dict__()
if not data.get('tags'):
data['tags'] = []
data = [data]
for d in data:
d = Indicator(**d)
if d.indicator in ["", 'localhost', 'example.com']:
continue
if self.exclude.get(d.provider):
for t in d.tags:
if t in self.exclude[d.provider]:
logger.debug('skipping: {}'.format(d.indicator))
for p in plugins:
if p.is_advanced:
if not HUNTER_ADVANCED:
continue
try:
p.process(d, router)
except Exception as e:
logger.error(e)
logger.error('[{}] giving up on: {}'.format(p, d))
def start(self):
router = Client(remote=self.router,
token=self.token,
nowait=True,
autoclose=False)
plugins = self._load_plugins()
socket = zmq.Context().socket(zmq.PULL)
socket.SNDTIMEO = SNDTIMEO
socket.set_hwm(ZMQ_HWM)
logger.debug('connecting to {}'.format(self.hunters))
socket.connect(self.hunters)
logger.debug('starting hunter')
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
while not self.exit.is_set():
try:
s = dict(poller.poll(1000))
except SystemExit or KeyboardInterrupt:
break
if socket not in s:
continue
id, token, mtype, data = Msg().recv(socket)
data = json.loads(data)
if isinstance(data, dict):
if not data.get('indicator'):
continue
if not data.get('itype'):
try:
data = Indicator(
indicator=data['indicator'],
tags='search',
confidence=10,
group='everyone',
tlp='amber',
).__dict__()
except InvalidIndicator:
logger.debug('skipping invalid indicator: {}'.format(
data['indicator']))
continue
if not data.get('tags'):
data['tags'] = []
data = [data]
token = json.loads(token)
for d in data:
d = Indicator(**d)
if d.confidence < HUNTER_MIN_CONFIDENCE:
continue
# prevent hunter recursion if disabled
if not HUNTER_RECURSION and d.tags and 'hunter' in d.tags:
continue
if d.indicator in ["", 'localhost', 'example.com']:
continue
if self.exclude.get(d.provider):
for t in d.tags:
if t in self.exclude[d.provider]:
logger.debug('skipping: {}'.format(d.indicator))
continue
for p in plugins:
if p.is_advanced:
if not HUNTER_ADVANCED:
continue
try:
p.process(i=d, router=router, user_token=token)
except Exception as e:
logger.error(e)
logger.error('[{}] giving up on: {}'.format(p, d))
def get(self):
filters = {}
start = time.time()
id = g.sid
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if len(filters) == 0:
return jsonify_unknown(
'invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)',
400)
if 'itype' not in filters:
return jsonify_unknown(
'missing itype filter (ipv4, fqdn, url, etc...)', 400)
# test to make sure feed type exists
feed_type = feed_factory(filters['itype'])
if feed_type is None:
err = "invalid feed itype: {}, valid types are [{}]".format(
filters['itype'], '|'.join(FEED_PLUGINS))
return jsonify_unknown(err, 400)
if not filters.get('reporttime'):
if not filters.get('days'):
if not filters.get('itype'):
filters['days'] = DAYS_SHORT
else:
filters['days'] = FEED_DAYS[filters['itype']]
if not filters.get('limit'):
filters['limit'] = FEEDS_LIMIT
if current_app.config.get('dummy'):
if current_app.config.get('feed'):
r = DummyClient(remote, pull_token()).indicators_search(
filters,
decode=True,
test_data=current_app.config['feed']['data'],
test_wl=current_app.config['feed']['wl'])
else:
r = DummyClient(remote,
pull_token()).indicators_search(filters)
return jsonify_success(r)
else:
logger.debug('%s building feed' % id)
logger.debug('%s getting dataset' % id)
try:
r = Client(remote, pull_token()).indicators_search(filters)
except AuthError:
return jsonify_unauth()
except RuntimeError as e:
return jsonify_unknown('search failed', 403)
except InvalidSearch as e:
logger.error(e)
return jsonify_unknown('invalid search', 400)
except Exception as e:
logger.error(e)
return jsonify_unknown(msg='search failed')
r = aggregate(r)
wl_filters = copy.deepcopy(filters)
# whitelists are typically updated 1/month so we should catch those
# esp for IP addresses
if not wl_filters.get('days') or int(wl_filters['days']) < 45:
wl_filters['days'] = 45
if wl_filters.get('reporttime'):
del wl_filters['reporttime']
wl_filters['tags'] = 'whitelist'
wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE
wl_filters['nolog'] = True
wl_filters['limit'] = FEEDS_WHITELIST_LIMIT
logger.debug('gathering whitelist..')
if current_app.config.get('feed') and current_app.config.get(
'feed').get('wl'):
wl = current_app.config.get('feed').get('wl')
else:
try:
wl = Client(remote, pull_token()).indicators_search(wl_filters)
except Exception as e:
logger.error(e)
return jsonify_unknown('feed query failed', 503)
logger.debug('%s aggregating' % id)
wl = aggregate(wl)
f = feed_factory(filters['itype'])
logger.debug('%s merging' % id)
r = f().process(r, wl)
logger.debug('%s done: %s' % (id, str((time.time() - start))))
response = jsonify({"message": "success", "data": r})
response.status_code = 200
return response
def client():
yield Client()
def tokens():
cli = Client(remote, pull_token())
if request.method == 'DELETE':
try:
r = cli.tokens_delete(request.data)
except Exception as e:
logger.error(e)
response = jsonify({"message": "failed", "data": []})
response.status_code = 503
else:
response = jsonify({'message': 'success', 'data': r})
response.status_code = 200
elif request.method == 'POST':
if request.data:
try:
r = cli.tokens_create(request.data)
except AuthError:
response = jsonify({
'message': 'admin privs required',
'data': []
})
response.status_code = 401
except Exception as e:
logger.error(e)
response = jsonify({'message': 'create failed', 'data': []})
response.status_code = 503
else:
if r:
response = jsonify({'message': 'success', 'data': r})
response.status_code = 200
else:
response = jsonify({
'message': 'admin privs required',
'data': []
})
response.status_code = 401
else:
response = jsonify({'message': 'create failed', 'data': []})
response.status_code = 400
elif request.method == 'PATCH':
try:
r = cli.tokens_edit(request.data)
except AuthError:
response = jsonify({'message': 'admin privs required', 'data': []})
response.status_code = 401
except Exception as e:
logger.error(e)
import traceback
traceback.print_exc()
response = jsonify({'message': 'create failed', 'data': []})
response.status_code = 503
else:
if r:
response = jsonify({'message': 'success', 'data': r})
response.status_code = 200
else:
response = jsonify({
'message': 'admin privs required',
'data': []
})
response.status_code = 401
else:
filters = {}
for f in TOKEN_FILTERS:
filters[f] = request.args.get(f)
try:
r = cli.tokens_search(filters)
except AuthError:
response = jsonify({"message": "failed", "data": []})
response.status_code = 401
except Exception as e:
logger.error(e)
response = jsonify({"message": "failed", "data": []})
response.status_code = 503
else:
response = jsonify({'message': 'success', 'data': r})
response.status_code = 200
return response
def get(self):
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if len(filters) == 0:
return jsonify_unknown('invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)', 400)
if 'itype' not in filters:
return jsonify_unknown('missing itype filter (ipv4, fqdn, url, etc...)', 400)
# test to make sure feed type exists
feed_type = feed_factory(filters['itype'])
if feed_type is None:
err = "invalid feed itype: {}, valid types are [{}]".format(filters['itype'], '|'.join(FEED_PLUGINS))
return jsonify_unknown(err, 400)
if not filters.get('days'):
filters['days'] = FEEDS_DAYS
if not filters.get('limit'):
filters['limit'] = FEEDS_LIMIT
if current_app.config.get('dummy'):
if current_app.config.get('feed'):
r = DummyClient(remote, pull_token()).indicators_search(filters,
decode=True,
test_data=current_app.config['feed']['data'],
test_wl=current_app.config['feed']['wl'])
else:
r = DummyClient(remote, pull_token()).indicators_search(filters)
return jsonify_success(r)
else:
try:
r = Client(remote, pull_token()).indicators_search(filters)
except AuthError:
return jsonify_unauth()
except RuntimeError as e:
return jsonify_unknown('search failed', 403)
except InvalidSearch as e:
logger.error(e)
return jsonify_unknown('invalid search', 400)
except Exception as e:
logger.error(e)
return jsonify_unknown(msg='search failed')
r = aggregate(r)
wl_filters = copy.deepcopy(filters)
wl_filters['tags'] = 'whitelist'
wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE
wl_filters['nolog'] = True
wl_filters['limit'] = FEEDS_WHITELIST_LIMIT
if current_app.config.get('feed').get('wl'):
wl = current_app.config.get('feed').get('wl')
else:
try:
wl = Client(remote, pull_token()).indicators_search(wl_filters)
except Exception as e:
logger.error(e)
return jsonify_unknown('feed query failed', 503)
wl = aggregate(wl)
f = feed_factory(filters['itype'])
r = f().process(r, wl)
response = jsonify({
"message": "success",
"data": r
})
response.status_code = 200
return response
def search():
"""
Search controller
:param str q: query term (ex: example.org, 1.2.3.4, 1.2.3.0/24)
:param dict filters: query filters
:param int limit: limit search results (reporttime desc)
:return: { 'message': 'success', 'data': [] }
"""
remote = ROUTER_ADDR
if app.config.get('CIF_ROUTER_ADDR'):
remote = app.config['CIF_ROUTER_ADDR']
filters = {}
for k in VALID_FILTERS:
if request.args.get(k):
filters[k] = request.args.get(k)
if request.args.get('q'):
filters['indicator'] = request.args.get('q')
try:
if request_v2():
if app.config.get('dummy'):
r = DummyClient(remote,
pull_token()).indicators_search(filters)
else:
r = Client(remote, pull_token()).indicators_search(filters)
for rr in r:
rr['observable'] = rr['indicator']
del rr['indicator']
if rr.get('itype'):
rr['otype'] = rr['itype']
del rr['itype']
response = jsonify({'message': 'success', 'data': r})
else:
if app.config.get('dummy'):
r = DummyClient(remote,
pull_token()).indicators_search(filters,
decode=False)
else:
r = Client(remote,
pull_token()).indicators_search(filters,
decode=False)
response = current_app.response_class(r,
mimetype='application/json')
if response_compress():
logger.debug('compressing')
response.data = compress(response.data)
response.status_code = 200
if isinstance(r, basestring):
if '"message":"unauthorized"' in r.decode(
'utf-8') and '"message":"unauthorized"' in r.decode(
'utf-8'):
response.status_code = 401
except AuthError as e:
response = jsonify({
'message': 'unauthorized',
'data': [],
'status': 'failed'
})
response.status_code = 401
except InvalidSearch as e:
logger.error(e)
response = jsonify({"message": "invalid search", "data": []})
response.status_code = 400
return response
def feed():
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if len(filters) == 0:
response = jsonify({
"message":
"invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)",
"data": []
})
response.status_code = 400
return response
# test to make sure feed type exists
feed_type = feed_factory(filters['itype'])
if feed_type is None:
response = jsonify({
"message":
"invalid feed itype: {}, valid types are [{}]".format(
filters['itype'], '|'.join(FEED_PLUGINS)),
"data": []
})
response.status_code = 400
return response
DAYS = request.args.get('days', 30)
LIMIT = request.args.get('limit', 50000)
now = arrow.utcnow()
filters['reporttimeend'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.replace(days=-DAYS)
filters['reporttime'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
try:
if app.config.get('dummy'):
r = DummyClient(remote, pull_token()).indicators_search(filters)
else:
r = Client(remote, pull_token()).indicators_search(filters)
except RuntimeError as e:
logger.error(e)
response = jsonify({"message": "search failed", "data": []})
response.status_code = 403
except InvalidSearch as e:
logger.error(e)
response = jsonify({"message": "invalid search", "data": []})
response.status_code = 400
else:
r = aggregate(r)
wl_filters = copy.deepcopy(filters)
wl_filters['tags'] = 'whitelist'
wl_filters['confidence'] = 25
now = arrow.utcnow()
now = now.replace(days=-DAYS)
wl_filters['reporttime'] = '{}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
wl_filters['nolog'] = True
wl_filters['limit'] = 25000
if app.config.get('dummy'):
wl = DummyClient(remote,
pull_token()).indicators_search(wl_filters)
else:
wl = Client(remote, pull_token()).indicators_search(wl_filters)
wl = aggregate(wl)
f = feed_factory(filters['itype'])
r = f().process(r, wl)
response = jsonify({"message": "success", "data": r})
if response_compress():
response.data = compress(response.data)
response.status_code = 200
return response
def get(self):
filters = {}
start = time.time()
id = g.sid
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if len(filters) == 0:
return jsonify_unknown(
'invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)',
400)
if 'itype' not in filters:
return jsonify_unknown(
'missing itype filter (ipv4, fqdn, url, etc...)', 400)
if filters.get('tags'):
if 'whitelist' in filters.get('tags'):
return jsonify_unknown(
'Invalid filter: tag "whitelist" is invalid for a feed. To find allow-listed indicators, perform a regular search rather than a feed pull',
400)
else:
# add a negation to ensure our search doesn't include allowlisted items
filters['tags'] += ',!whitelist'
else:
filters['tags'] = '!whitelist'
# test to make sure feed type exists
feed_type = feed_factory(filters['itype'])
if feed_type is None:
err = "invalid feed itype: {}, valid types are [{}]".format(
filters['itype'], '|'.join(FEED_PLUGINS))
return jsonify_unknown(err, 400)
if not filters.get('reporttime'):
if not filters.get('days'):
if not filters.get('itype'):
filters['days'] = DAYS_SHORT
else:
filters['days'] = FEED_DAYS[filters['itype']]
if not filters.get('limit'):
filters['limit'] = FEEDS_LIMIT
if current_app.config.get('dummy'):
if current_app.config.get('feed'):
r = DummyClient(remote, pull_token()).indicators_search(
filters,
decode=True,
test_data=current_app.config['feed']['data'],
test_wl=current_app.config['feed']['wl'])
else:
r = DummyClient(remote,
pull_token()).indicators_search(filters)
return jsonify_success(r)
else:
logger.debug('%s building feed' % id)
logger.debug('%s getting dataset' % id)
try:
r = Client(remote, pull_token()).indicators_search(filters)
except AuthError:
return jsonify_unauth()
except RuntimeError as e:
return jsonify_unknown('search failed', 403)
except InvalidSearch as e:
logger.error(e)
return jsonify_unknown('invalid search', 400)
except Exception as e:
logger.error(e)
return jsonify_unknown(msg='search failed')
r = aggregate(r)
wl_filters = copy.deepcopy(filters)
# whitelists are typically updated 1/month so we should catch those
# esp for IP addresses
if not wl_filters.get('days') or int(wl_filters['days']) < 45:
wl_filters['days'] = 45
if wl_filters.get('reporttime'):
del wl_filters['reporttime']
wl_filters['tags'] = 'whitelist'
wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE
wl_filters['nolog'] = True
wl_filters['limit'] = FEEDS_WHITELIST_LIMIT
# remove provider from wl_filters if exists (we don't want to narrow wl scope by provider)
wl_filters.pop('provider', None)
logger.debug('gathering whitelist..')
if current_app.config.get('feed') and current_app.config.get(
'feed').get('wl'):
wl = current_app.config.get('feed').get('wl')
else:
try:
wl = Client(internal_remote,
pull_token()).indicators_search(wl_filters)
except Exception as e:
logger.error(e)
return jsonify_unknown('feed query failed', 503)
logger.debug('%s aggregating' % id)
wl = aggregate(wl)
f = feed_factory(filters['itype'])
logger.debug('%s merging' % id)
r = f().process(r, wl)
logger.debug('%s done: %s' % (id, str((time.time() - start))))
# manually make flask Response obj rather than use jsonify
# to take advantage of ujson speed for large str
response = make_response(json.dumps({"message": "success", "data": r}))
response.headers['Content-Type'] = 'application/json'
response.status_code = 200
return response
def feed():
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
DAYS = request.args.get('days', 30)
LIMIT = request.args.get('limit', 50000)
now = arrow.utcnow()
filters['reporttimeend'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.replace(days=-DAYS)
filters['reporttime'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
try:
if app.config.get('dummy'):
r = DummyClient(remote, pull_token()).indicators_search(filters)
else:
r = Client(remote, pull_token()).indicators_search(filters)
except RuntimeError as e:
logger.error(e)
response = jsonify({"message": "search failed", "data": []})
response.status_code = 403
except InvalidSearch as e:
logger.error(e)
response = jsonify({"message": "invalid search", "data": []})
response.status_code = 400
else:
r = aggregate(r)
wl_filters = copy.deepcopy(filters)
wl_filters['tags'] = 'whitelist'
wl_filters['confidence'] = 25
now = arrow.utcnow()
now = now.replace(days=-DAYS)
wl_filters['reporttime'] = '{}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
wl_filters['nolog'] = True
wl_filters['limit'] = 25000
if app.config.get('dummy'):
wl = DummyClient(remote,
pull_token()).indicators_search(wl_filters)
else:
wl = Client(remote, pull_token()).indicators_search(wl_filters)
wl = aggregate(wl)
from .feed import factory as feed_factory
f = feed_factory(filters['itype'])
r = f().process(r, wl)
response = jsonify({"message": "success", "data": r})
if response_compress():
response.data = compress(response.data)
response.status_code = 200
return response
def indicators():
"""
GET/POST the Indicators Controller
:return: { 'message': '{success|failure}', 'data': [] }
"""
if request.method == 'GET':
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
try:
if app.config.get('dummy'):
r = DummyClient(remote, pull_token()).filter(
filters=filters, limit=request.args.get('limit'))
else:
r = Client(remote, pull_token()).filter(
filters=filters, limit=request.args.get('limit'))
except RuntimeError as e:
logger.error(e)
response = jsonify({"message": "search failed", "data": []})
response.status_code = 403
except InvalidSearch as e:
logger.error(e)
response = jsonify({"message": "invalid search", "data": []})
response.status_code = 400
else:
response = jsonify({"message": "success", "data": r})
if request.args.get('gzip'):
response.data = compress(response.data)
response.status_code = 200
else:
fireball = False
if request.headers.get('Content-Length'):
logger.debug('content-length: %s' %
request.headers['Content-Length'])
if int(request.headers['Content-Length']) > 10000:
logger.info('fireball mode')
fireball = True
try:
data = request.data.decode('utf-8')
r = Client(remote,
pull_token()).indicators_create(data, fireball=fireball)
except RuntimeError as e:
logger.error(e)
response = jsonify({"message": "submission failed", "data": []})
response.status_code = 422
except TimeoutError as e:
logger.error(e)
response = jsonify({"message": "submission failed", "data": []})
response.status_code = 408
except Exception as e:
logger.error(e)
response = jsonify({"message": "submission failed", "data": []})
response.status_code = 422
else:
response = jsonify({"message": "success", "data": r})
response.status_code = 201
return response
def test_client_zmq():
cli = Client(ROUTER_ADDR, '12345')
assert cli.remote == ROUTER_ADDR
assert cli.token == '12345'