def get(self):
remote = ROUTER_ADDR
if current_app.config.get('CIF_ROUTER_ADDR'):
remote = current_app.config['CIF_ROUTER_ADDR']
write = request.args.get('write', None)
if current_app.config.get('dummy'):
r = DummyClient(remote, pull_token()).ping(write=write)
return jsonify_success(r)
try:
with Client(remote, pull_token()) as cli:
r = cli.ping(write=write)
except TimeoutError:
return jsonify_unknown(msg='timeout', code=408)
except AuthError:
return jsonify_unauth()
if not r:
return jsonify_unknown(503)
return jsonify_success(r)
def ping():
"""
Ping the router interface
:return: { 'message': 'success', 'data': '<timestamp>' }
"""
remote = ROUTER_ADDR
if app.config.get('CIF_ROUTER_ADDR'):
remote = app.config['CIF_ROUTER_ADDR']
write = request.args.get('write', None)
if app.config.get('dummy'):
r = DummyClient(remote, pull_token()).ping(write=write)
else:
r = Client(remote, pull_token()).ping(write=write)
resp = jsonify({"message": "success", "data": r})
if not r:
resp = jsonify({
'message': 'failed',
'data': [],
})
resp.status_code = 401
return resp
def search():
"""
Search controller
:param str q: query term (ex: example.org, 1.2.3.4, 1.2.3.0/24)
:param dict filters: query filters
:param int limit: limit search results (reporttime desc)
:return: { 'message': 'success', 'data': [] }
"""
remote = ROUTER_ADDR
if app.config.get('CIF_ROUTER_ADDR'):
remote = app.config['CIF_ROUTER_ADDR']
filters = {}
for k in VALID_FILTERS:
if request.args.get(k):
filters[k] = request.args.get(k)
if request.args.get('q'):
filters['indicator'] = request.args.get('q')
try:
if app.config.get('dummy'):
r = DummyClient(remote, pull_token()).indicators_search(filters)
else:
r = Client(remote, pull_token()).indicators_search(filters)
if request_v2():
for rr in r:
rr['observable'] = rr['indicator']
del rr['indicator']
if rr.get('itype'):
rr['otype'] = rr['itype']
del rr['itype']
response = jsonify({"message": "success", "data": r})
if response_compress():
response.data = compress(response.data)
response.status_code = 200
except AuthError as e:
response = jsonify({
'message': 'unauthorized',
'data': [],
'status': 'failed'
})
response.status_code = 401
except InvalidSearch as e:
logger.error(e)
response = jsonify({"message": "invalid search", "data": []})
response.status_code = 400
return response
def get(self):
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if request.args.get('q'):
filters['indicator'] = request.args.get('q')
if request.args.get('confidence'):
filters['confidence'] = request.args.get('confidence')
if request.args.get('provider'):
filters['provider'] = request.args.get('provider')
if request.args.get('group'):
filters['group'] = request.args.get('group')
if request.args.get('tags'):
filters['tags'] = request.args.get('tags')
if request.args.get('lasttime'):
filters['lasttime'] = request.args.get('lasttime')
if current_app.config.get('dummy'):
r = DummyClient(remote, pull_token()).indicators_search(filters)
return jsonify_success(r)
try:
with Client(remote, pull_token()) as cli:
r = cli.indicators_search(filters, decode=False)
except RuntimeError as e:
logger.error(e)
return jsonify_unknown(msg='search failed')
except InvalidSearch as e:
return jsonify_unknown(msg='invalid search', code=400)
except AuthError:
return jsonify_unauth()
except Exception as e:
logger.error(e)
return jsonify_unknown(
msg='search failed, system may be too busy, check back later')
response = current_app.response_class(r, mimetype='application/json')
if isinstance(r, basestring):
if '"message":"unauthorized"' in r and '"message":"unauthorized"' in r:
response.status_code = 401
return response
return response
def indicators():
"""
GET/POST the Indicators Controller
:return: { 'message': '{success|failure}', 'data': [] }
"""
if request.method == 'GET':
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
try:
if app.config.get('dummy'):
r = DummyClient(remote, pull_token()).filter(
filters=filters, limit=request.args.get('limit'))
else:
r = Client(remote, pull_token()).filter(
filters=filters, limit=request.args.get('limit'))
except RuntimeError as e:
logger.error(e)
response = jsonify({"message": "search failed", "data": []})
response.status_code = 403
else:
response = jsonify({"message": "success", "data": r})
response.status_code = 200
else:
try:
logger.debug(request.data)
r = Client(remote, pull_token()).indicators_create(request.data)
except RuntimeError as e:
logger.error(e)
response = jsonify({"message": "submission failed", "data": []})
response.status_code = 422
else:
response = jsonify({"message": "success", "data": r})
response.status_code = 201
return response
def get(self):
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if request.args.get('q'):
filters['indicator'] = request.args.get('q')
if current_app.config.get('dummy'):
r = DummyClient(remote, pull_token()).indicators_search(filters)
return jsonify_success(r)
try:
r = Client(remote, pull_token()).indicators_search(filters, decode=False)
except RuntimeError as e:
logger.error(e)
return jsonify_unknown(msg='search failed', code=403)
except InvalidSearch as e:
return jsonify_unknown(msg='invalid search', code=400)
except AuthError:
return jsonify_unauth()
response = current_app.response_class(r, mimetype='application/json')
if isinstance(r, basestring):
if '"message":"unauthorized"' in r and '"message":"unauthorized"' in r:
response.status_code = 401
return response
if response_compress():
response.data = compress(response.data)
return response
def get(self):
filters = {}
start = time.time()
id = g.sid
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if len(filters) == 0:
return jsonify_unknown(
'invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)',
400)
if 'itype' not in filters:
return jsonify_unknown(
'missing itype filter (ipv4, fqdn, url, etc...)', 400)
# test to make sure feed type exists
feed_type = feed_factory(filters['itype'])
if feed_type is None:
err = "invalid feed itype: {}, valid types are [{}]".format(
filters['itype'], '|'.join(FEED_PLUGINS))
return jsonify_unknown(err, 400)
if not filters.get('reporttime'):
if not filters.get('days'):
if not filters.get('itype'):
filters['days'] = DAYS_SHORT
else:
filters['days'] = FEED_DAYS[filters['itype']]
if not filters.get('limit'):
filters['limit'] = FEEDS_LIMIT
if current_app.config.get('dummy'):
if current_app.config.get('feed'):
r = DummyClient(remote, pull_token()).indicators_search(
filters,
decode=True,
test_data=current_app.config['feed']['data'],
test_wl=current_app.config['feed']['wl'])
else:
r = DummyClient(remote,
pull_token()).indicators_search(filters)
return jsonify_success(r)
else:
logger.debug('%s building feed' % id)
logger.debug('%s getting dataset' % id)
try:
r = Client(remote, pull_token()).indicators_search(filters)
except AuthError:
return jsonify_unauth()
except RuntimeError as e:
return jsonify_unknown('search failed', 403)
except InvalidSearch as e:
logger.error(e)
return jsonify_unknown('invalid search', 400)
except Exception as e:
logger.error(e)
return jsonify_unknown(msg='search failed')
r = aggregate(r)
wl_filters = copy.deepcopy(filters)
# whitelists are typically updated 1/month so we should catch those
# esp for IP addresses
if not wl_filters.get('days') or int(wl_filters['days']) < 45:
wl_filters['days'] = 45
if wl_filters.get('reporttime'):
del wl_filters['reporttime']
wl_filters['tags'] = 'whitelist'
wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE
wl_filters['nolog'] = True
wl_filters['limit'] = FEEDS_WHITELIST_LIMIT
logger.debug('gathering whitelist..')
if current_app.config.get('feed') and current_app.config.get(
'feed').get('wl'):
wl = current_app.config.get('feed').get('wl')
else:
try:
wl = Client(remote, pull_token()).indicators_search(wl_filters)
except Exception as e:
logger.error(e)
return jsonify_unknown('feed query failed', 503)
logger.debug('%s aggregating' % id)
wl = aggregate(wl)
f = feed_factory(filters['itype'])
logger.debug('%s merging' % id)
r = f().process(r, wl)
logger.debug('%s done: %s' % (id, str((time.time() - start))))
response = jsonify({"message": "success", "data": r})
response.status_code = 200
return response
def get(self):
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if len(filters) == 0:
return jsonify_unknown('invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)', 400)
if 'itype' not in filters:
return jsonify_unknown('missing itype filter (ipv4, fqdn, url, etc...)', 400)
# test to make sure feed type exists
feed_type = feed_factory(filters['itype'])
if feed_type is None:
err = "invalid feed itype: {}, valid types are [{}]".format(filters['itype'], '|'.join(FEED_PLUGINS))
return jsonify_unknown(err, 400)
if not filters.get('days'):
filters['days'] = FEEDS_DAYS
if not filters.get('limit'):
filters['limit'] = FEEDS_LIMIT
if current_app.config.get('dummy'):
if current_app.config.get('feed'):
r = DummyClient(remote, pull_token()).indicators_search(filters,
decode=True,
test_data=current_app.config['feed']['data'],
test_wl=current_app.config['feed']['wl'])
else:
r = DummyClient(remote, pull_token()).indicators_search(filters)
return jsonify_success(r)
else:
try:
r = Client(remote, pull_token()).indicators_search(filters)
except AuthError:
return jsonify_unauth()
except RuntimeError as e:
return jsonify_unknown('search failed', 403)
except InvalidSearch as e:
logger.error(e)
return jsonify_unknown('invalid search', 400)
except Exception as e:
logger.error(e)
return jsonify_unknown(msg='search failed')
r = aggregate(r)
wl_filters = copy.deepcopy(filters)
wl_filters['tags'] = 'whitelist'
wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE
wl_filters['nolog'] = True
wl_filters['limit'] = FEEDS_WHITELIST_LIMIT
if current_app.config.get('feed').get('wl'):
wl = current_app.config.get('feed').get('wl')
else:
try:
wl = Client(remote, pull_token()).indicators_search(wl_filters)
except Exception as e:
logger.error(e)
return jsonify_unknown('feed query failed', 503)
wl = aggregate(wl)
f = feed_factory(filters['itype'])
r = f().process(r, wl)
response = jsonify({
"message": "success",
"data": r
})
response.status_code = 200
return response
def get(self):
filters = {}
start = time.time()
id = g.sid
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if len(filters) == 0:
return jsonify_unknown(
'invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)',
400)
if 'itype' not in filters:
return jsonify_unknown(
'missing itype filter (ipv4, fqdn, url, etc...)', 400)
if filters.get('tags'):
if 'whitelist' in filters.get('tags'):
return jsonify_unknown(
'Invalid filter: tag "whitelist" is invalid for a feed. To find allow-listed indicators, perform a regular search rather than a feed pull',
400)
else:
# add a negation to ensure our search doesn't include allowlisted items
filters['tags'] += ',!whitelist'
else:
filters['tags'] = '!whitelist'
# test to make sure feed type exists
feed_type = feed_factory(filters['itype'])
if feed_type is None:
err = "invalid feed itype: {}, valid types are [{}]".format(
filters['itype'], '|'.join(FEED_PLUGINS))
return jsonify_unknown(err, 400)
if not filters.get('reporttime'):
if not filters.get('days'):
if not filters.get('itype'):
filters['days'] = DAYS_SHORT
else:
filters['days'] = FEED_DAYS[filters['itype']]
if not filters.get('limit'):
filters['limit'] = FEEDS_LIMIT
if current_app.config.get('dummy'):
if current_app.config.get('feed'):
r = DummyClient(remote, pull_token()).indicators_search(
filters,
decode=True,
test_data=current_app.config['feed']['data'],
test_wl=current_app.config['feed']['wl'])
else:
r = DummyClient(remote,
pull_token()).indicators_search(filters)
return jsonify_success(r)
else:
logger.debug('%s building feed' % id)
logger.debug('%s getting dataset' % id)
try:
r = Client(remote, pull_token()).indicators_search(filters)
except AuthError:
return jsonify_unauth()
except RuntimeError as e:
return jsonify_unknown('search failed', 403)
except InvalidSearch as e:
logger.error(e)
return jsonify_unknown('invalid search', 400)
except Exception as e:
logger.error(e)
return jsonify_unknown(msg='search failed')
r = aggregate(r)
wl_filters = copy.deepcopy(filters)
# whitelists are typically updated 1/month so we should catch those
# esp for IP addresses
if not wl_filters.get('days') or int(wl_filters['days']) < 45:
wl_filters['days'] = 45
if wl_filters.get('reporttime'):
del wl_filters['reporttime']
wl_filters['tags'] = 'whitelist'
wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE
wl_filters['nolog'] = True
wl_filters['limit'] = FEEDS_WHITELIST_LIMIT
# remove provider from wl_filters if exists (we don't want to narrow wl scope by provider)
wl_filters.pop('provider', None)
logger.debug('gathering whitelist..')
if current_app.config.get('feed') and current_app.config.get(
'feed').get('wl'):
wl = current_app.config.get('feed').get('wl')
else:
try:
wl = Client(internal_remote,
pull_token()).indicators_search(wl_filters)
except Exception as e:
logger.error(e)
return jsonify_unknown('feed query failed', 503)
logger.debug('%s aggregating' % id)
wl = aggregate(wl)
f = feed_factory(filters['itype'])
logger.debug('%s merging' % id)
r = f().process(r, wl)
logger.debug('%s done: %s' % (id, str((time.time() - start))))
# manually make flask Response obj rather than use jsonify
# to take advantage of ujson speed for large str
response = make_response(json.dumps({"message": "success", "data": r}))
response.headers['Content-Type'] = 'application/json'
response.status_code = 200
return response
def indicators():
"""
GET/POST the Indicators Controller
:return: { 'message': '{success|failure}', 'data': [] }
"""
if request.method == 'GET':
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
try:
if app.config.get('dummy'):
r = DummyClient(remote, pull_token()).filter(
filters=filters, limit=request.args.get('limit'))
else:
r = Client(remote, pull_token()).filter(
filters=filters, limit=request.args.get('limit'))
except RuntimeError as e:
logger.error(e)
response = jsonify({"message": "search failed", "data": []})
response.status_code = 403
except InvalidSearch as e:
logger.error(e)
response = jsonify({"message": "invalid search", "data": []})
response.status_code = 400
else:
response = jsonify({"message": "success", "data": r})
if request.args.get('gzip'):
response.data = compress(response.data)
response.status_code = 200
else:
fireball = False
if request.headers.get('Content-Length'):
logger.debug('content-length: %s' %
request.headers['Content-Length'])
if int(request.headers['Content-Length']) > 10000:
logger.info('fireball mode')
fireball = True
try:
data = request.data.decode('utf-8')
r = Client(remote,
pull_token()).indicators_create(data, fireball=fireball)
except RuntimeError as e:
logger.error(e)
response = jsonify({"message": "submission failed", "data": []})
response.status_code = 422
except TimeoutError as e:
logger.error(e)
response = jsonify({"message": "submission failed", "data": []})
response.status_code = 408
except Exception as e:
logger.error(e)
response = jsonify({"message": "submission failed", "data": []})
response.status_code = 422
else:
response = jsonify({"message": "success", "data": r})
response.status_code = 201
return response
def feed():
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
DAYS = request.args.get('days', 30)
LIMIT = request.args.get('limit', 50000)
now = arrow.utcnow()
filters['reporttimeend'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.replace(days=-DAYS)
filters['reporttime'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
try:
if app.config.get('dummy'):
r = DummyClient(remote, pull_token()).indicators_search(filters)
else:
r = Client(remote, pull_token()).indicators_search(filters)
except RuntimeError as e:
logger.error(e)
response = jsonify({"message": "search failed", "data": []})
response.status_code = 403
except InvalidSearch as e:
logger.error(e)
response = jsonify({"message": "invalid search", "data": []})
response.status_code = 400
else:
r = aggregate(r)
wl_filters = copy.deepcopy(filters)
wl_filters['tags'] = 'whitelist'
wl_filters['confidence'] = 25
now = arrow.utcnow()
now = now.replace(days=-DAYS)
wl_filters['reporttime'] = '{}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
wl_filters['nolog'] = True
wl_filters['limit'] = 25000
if app.config.get('dummy'):
wl = DummyClient(remote,
pull_token()).indicators_search(wl_filters)
else:
wl = Client(remote, pull_token()).indicators_search(wl_filters)
wl = aggregate(wl)
from .feed import factory as feed_factory
f = feed_factory(filters['itype'])
r = f().process(r, wl)
response = jsonify({"message": "success", "data": r})
if response_compress():
response.data = compress(response.data)
response.status_code = 200
return response
def feed():
filters = {}
for f in VALID_FILTERS:
if request.args.get(f):
filters[f] = request.args.get(f)
if len(filters) == 0:
response = jsonify({
"message":
"invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)",
"data": []
})
response.status_code = 400
return response
# test to make sure feed type exists
feed_type = feed_factory(filters['itype'])
if feed_type is None:
response = jsonify({
"message":
"invalid feed itype: {}, valid types are [{}]".format(
filters['itype'], '|'.join(FEED_PLUGINS)),
"data": []
})
response.status_code = 400
return response
DAYS = request.args.get('days', 30)
LIMIT = request.args.get('limit', 50000)
now = arrow.utcnow()
filters['reporttimeend'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.replace(days=-DAYS)
filters['reporttime'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
try:
if app.config.get('dummy'):
r = DummyClient(remote, pull_token()).indicators_search(filters)
else:
r = Client(remote, pull_token()).indicators_search(filters)
except RuntimeError as e:
logger.error(e)
response = jsonify({"message": "search failed", "data": []})
response.status_code = 403
except InvalidSearch as e:
logger.error(e)
response = jsonify({"message": "invalid search", "data": []})
response.status_code = 400
else:
r = aggregate(r)
wl_filters = copy.deepcopy(filters)
wl_filters['tags'] = 'whitelist'
wl_filters['confidence'] = 25
now = arrow.utcnow()
now = now.replace(days=-DAYS)
wl_filters['reporttime'] = '{}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
wl_filters['nolog'] = True
wl_filters['limit'] = 25000
if app.config.get('dummy'):
wl = DummyClient(remote,
pull_token()).indicators_search(wl_filters)
else:
wl = Client(remote, pull_token()).indicators_search(wl_filters)
wl = aggregate(wl)
f = feed_factory(filters['itype'])
r = f().process(r, wl)
response = jsonify({"message": "success", "data": r})
if response_compress():
response.data = compress(response.data)
response.status_code = 200
return response
