def get(self): remote = ROUTER_ADDR if current_app.config.get('CIF_ROUTER_ADDR'): remote = current_app.config['CIF_ROUTER_ADDR'] write = request.args.get('write', None) if current_app.config.get('dummy'): r = DummyClient(remote, pull_token()).ping(write=write) return jsonify_success(r) try: with Client(remote, pull_token()) as cli: r = cli.ping(write=write) except TimeoutError: return jsonify_unknown(msg='timeout', code=408) except AuthError: return jsonify_unauth() if not r: return jsonify_unknown(503) return jsonify_success(r)
def ping(): """ Ping the router interface :return: { 'message': 'success', 'data': '<timestamp>' } """ remote = ROUTER_ADDR if app.config.get('CIF_ROUTER_ADDR'): remote = app.config['CIF_ROUTER_ADDR'] write = request.args.get('write', None) if app.config.get('dummy'): r = DummyClient(remote, pull_token()).ping(write=write) else: r = Client(remote, pull_token()).ping(write=write) resp = jsonify({"message": "success", "data": r}) if not r: resp = jsonify({ 'message': 'failed', 'data': [], }) resp.status_code = 401 return resp
def search(): """ Search controller :param str q: query term (ex: example.org, 1.2.3.4, 1.2.3.0/24) :param dict filters: query filters :param int limit: limit search results (reporttime desc) :return: { 'message': 'success', 'data': [] } """ remote = ROUTER_ADDR if app.config.get('CIF_ROUTER_ADDR'): remote = app.config['CIF_ROUTER_ADDR'] filters = {} for k in VALID_FILTERS: if request.args.get(k): filters[k] = request.args.get(k) if request.args.get('q'): filters['indicator'] = request.args.get('q') try: if app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters) else: r = Client(remote, pull_token()).indicators_search(filters) if request_v2(): for rr in r: rr['observable'] = rr['indicator'] del rr['indicator'] if rr.get('itype'): rr['otype'] = rr['itype'] del rr['itype'] response = jsonify({"message": "success", "data": r}) if response_compress(): response.data = compress(response.data) response.status_code = 200 except AuthError as e: response = jsonify({ 'message': 'unauthorized', 'data': [], 'status': 'failed' }) response.status_code = 401 except InvalidSearch as e: logger.error(e) response = jsonify({"message": "invalid search", "data": []}) response.status_code = 400 return response
def get(self): filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if request.args.get('q'): filters['indicator'] = request.args.get('q') if request.args.get('confidence'): filters['confidence'] = request.args.get('confidence') if request.args.get('provider'): filters['provider'] = request.args.get('provider') if request.args.get('group'): filters['group'] = request.args.get('group') if request.args.get('tags'): filters['tags'] = request.args.get('tags') if request.args.get('lasttime'): filters['lasttime'] = request.args.get('lasttime') if current_app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters) return jsonify_success(r) try: with Client(remote, pull_token()) as cli: r = cli.indicators_search(filters, decode=False) except RuntimeError as e: logger.error(e) return jsonify_unknown(msg='search failed') except InvalidSearch as e: return jsonify_unknown(msg='invalid search', code=400) except AuthError: return jsonify_unauth() except Exception as e: logger.error(e) return jsonify_unknown( msg='search failed, system may be too busy, check back later') response = current_app.response_class(r, mimetype='application/json') if isinstance(r, basestring): if '"message":"unauthorized"' in r and '"message":"unauthorized"' in r: response.status_code = 401 return response return response
def indicators(): """ GET/POST the Indicators Controller :return: { 'message': '{success|failure}', 'data': [] } """ if request.method == 'GET': filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) try: if app.config.get('dummy'): r = DummyClient(remote, pull_token()).filter( filters=filters, limit=request.args.get('limit')) else: r = Client(remote, pull_token()).filter( filters=filters, limit=request.args.get('limit')) except RuntimeError as e: logger.error(e) response = jsonify({"message": "search failed", "data": []}) response.status_code = 403 else: response = jsonify({"message": "success", "data": r}) response.status_code = 200 else: try: logger.debug(request.data) r = Client(remote, pull_token()).indicators_create(request.data) except RuntimeError as e: logger.error(e) response = jsonify({"message": "submission failed", "data": []}) response.status_code = 422 else: response = jsonify({"message": "success", "data": r}) response.status_code = 201 return response
def get(self): filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if request.args.get('q'): filters['indicator'] = request.args.get('q') if current_app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters) return jsonify_success(r) try: r = Client(remote, pull_token()).indicators_search(filters, decode=False) except RuntimeError as e: logger.error(e) return jsonify_unknown(msg='search failed', code=403) except InvalidSearch as e: return jsonify_unknown(msg='invalid search', code=400) except AuthError: return jsonify_unauth() response = current_app.response_class(r, mimetype='application/json') if isinstance(r, basestring): if '"message":"unauthorized"' in r and '"message":"unauthorized"' in r: response.status_code = 401 return response if response_compress(): response.data = compress(response.data) return response
def get(self): filters = {} start = time.time() id = g.sid for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if len(filters) == 0: return jsonify_unknown( 'invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)', 400) if 'itype' not in filters: return jsonify_unknown( 'missing itype filter (ipv4, fqdn, url, etc...)', 400) # test to make sure feed type exists feed_type = feed_factory(filters['itype']) if feed_type is None: err = "invalid feed itype: {}, valid types are [{}]".format( filters['itype'], '|'.join(FEED_PLUGINS)) return jsonify_unknown(err, 400) if not filters.get('reporttime'): if not filters.get('days'): if not filters.get('itype'): filters['days'] = DAYS_SHORT else: filters['days'] = FEED_DAYS[filters['itype']] if not filters.get('limit'): filters['limit'] = FEEDS_LIMIT if current_app.config.get('dummy'): if current_app.config.get('feed'): r = DummyClient(remote, pull_token()).indicators_search( filters, decode=True, test_data=current_app.config['feed']['data'], test_wl=current_app.config['feed']['wl']) else: r = DummyClient(remote, pull_token()).indicators_search(filters) return jsonify_success(r) else: logger.debug('%s building feed' % id) logger.debug('%s getting dataset' % id) try: r = Client(remote, pull_token()).indicators_search(filters) except AuthError: return jsonify_unauth() except RuntimeError as e: return jsonify_unknown('search failed', 403) except InvalidSearch as e: logger.error(e) return jsonify_unknown('invalid search', 400) except Exception as e: logger.error(e) return jsonify_unknown(msg='search failed') r = aggregate(r) wl_filters = copy.deepcopy(filters) # whitelists are typically updated 1/month so we should catch those # esp for IP addresses if not wl_filters.get('days') or int(wl_filters['days']) < 45: wl_filters['days'] = 45 if wl_filters.get('reporttime'): del wl_filters['reporttime'] wl_filters['tags'] = 'whitelist' wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE wl_filters['nolog'] = True wl_filters['limit'] = FEEDS_WHITELIST_LIMIT logger.debug('gathering whitelist..') if current_app.config.get('feed') and current_app.config.get( 'feed').get('wl'): wl = current_app.config.get('feed').get('wl') else: try: wl = Client(remote, pull_token()).indicators_search(wl_filters) except Exception as e: logger.error(e) return jsonify_unknown('feed query failed', 503) logger.debug('%s aggregating' % id) wl = aggregate(wl) f = feed_factory(filters['itype']) logger.debug('%s merging' % id) r = f().process(r, wl) logger.debug('%s done: %s' % (id, str((time.time() - start)))) response = jsonify({"message": "success", "data": r}) response.status_code = 200 return response
def get(self): filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if len(filters) == 0: return jsonify_unknown('invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)', 400) if 'itype' not in filters: return jsonify_unknown('missing itype filter (ipv4, fqdn, url, etc...)', 400) # test to make sure feed type exists feed_type = feed_factory(filters['itype']) if feed_type is None: err = "invalid feed itype: {}, valid types are [{}]".format(filters['itype'], '|'.join(FEED_PLUGINS)) return jsonify_unknown(err, 400) if not filters.get('days'): filters['days'] = FEEDS_DAYS if not filters.get('limit'): filters['limit'] = FEEDS_LIMIT if current_app.config.get('dummy'): if current_app.config.get('feed'): r = DummyClient(remote, pull_token()).indicators_search(filters, decode=True, test_data=current_app.config['feed']['data'], test_wl=current_app.config['feed']['wl']) else: r = DummyClient(remote, pull_token()).indicators_search(filters) return jsonify_success(r) else: try: r = Client(remote, pull_token()).indicators_search(filters) except AuthError: return jsonify_unauth() except RuntimeError as e: return jsonify_unknown('search failed', 403) except InvalidSearch as e: logger.error(e) return jsonify_unknown('invalid search', 400) except Exception as e: logger.error(e) return jsonify_unknown(msg='search failed') r = aggregate(r) wl_filters = copy.deepcopy(filters) wl_filters['tags'] = 'whitelist' wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE wl_filters['nolog'] = True wl_filters['limit'] = FEEDS_WHITELIST_LIMIT if current_app.config.get('feed').get('wl'): wl = current_app.config.get('feed').get('wl') else: try: wl = Client(remote, pull_token()).indicators_search(wl_filters) except Exception as e: logger.error(e) return jsonify_unknown('feed query failed', 503) wl = aggregate(wl) f = feed_factory(filters['itype']) r = f().process(r, wl) response = jsonify({ "message": "success", "data": r }) response.status_code = 200 return response
def get(self): filters = {} start = time.time() id = g.sid for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if len(filters) == 0: return jsonify_unknown( 'invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)', 400) if 'itype' not in filters: return jsonify_unknown( 'missing itype filter (ipv4, fqdn, url, etc...)', 400) if filters.get('tags'): if 'whitelist' in filters.get('tags'): return jsonify_unknown( 'Invalid filter: tag "whitelist" is invalid for a feed. To find allow-listed indicators, perform a regular search rather than a feed pull', 400) else: # add a negation to ensure our search doesn't include allowlisted items filters['tags'] += ',!whitelist' else: filters['tags'] = '!whitelist' # test to make sure feed type exists feed_type = feed_factory(filters['itype']) if feed_type is None: err = "invalid feed itype: {}, valid types are [{}]".format( filters['itype'], '|'.join(FEED_PLUGINS)) return jsonify_unknown(err, 400) if not filters.get('reporttime'): if not filters.get('days'): if not filters.get('itype'): filters['days'] = DAYS_SHORT else: filters['days'] = FEED_DAYS[filters['itype']] if not filters.get('limit'): filters['limit'] = FEEDS_LIMIT if current_app.config.get('dummy'): if current_app.config.get('feed'): r = DummyClient(remote, pull_token()).indicators_search( filters, decode=True, test_data=current_app.config['feed']['data'], test_wl=current_app.config['feed']['wl']) else: r = DummyClient(remote, pull_token()).indicators_search(filters) return jsonify_success(r) else: logger.debug('%s building feed' % id) logger.debug('%s getting dataset' % id) try: r = Client(remote, pull_token()).indicators_search(filters) except AuthError: return jsonify_unauth() except RuntimeError as e: return jsonify_unknown('search failed', 403) except InvalidSearch as e: logger.error(e) return jsonify_unknown('invalid search', 400) except Exception as e: logger.error(e) return jsonify_unknown(msg='search failed') r = aggregate(r) wl_filters = copy.deepcopy(filters) # whitelists are typically updated 1/month so we should catch those # esp for IP addresses if not wl_filters.get('days') or int(wl_filters['days']) < 45: wl_filters['days'] = 45 if wl_filters.get('reporttime'): del wl_filters['reporttime'] wl_filters['tags'] = 'whitelist' wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE wl_filters['nolog'] = True wl_filters['limit'] = FEEDS_WHITELIST_LIMIT # remove provider from wl_filters if exists (we don't want to narrow wl scope by provider) wl_filters.pop('provider', None) logger.debug('gathering whitelist..') if current_app.config.get('feed') and current_app.config.get( 'feed').get('wl'): wl = current_app.config.get('feed').get('wl') else: try: wl = Client(internal_remote, pull_token()).indicators_search(wl_filters) except Exception as e: logger.error(e) return jsonify_unknown('feed query failed', 503) logger.debug('%s aggregating' % id) wl = aggregate(wl) f = feed_factory(filters['itype']) logger.debug('%s merging' % id) r = f().process(r, wl) logger.debug('%s done: %s' % (id, str((time.time() - start)))) # manually make flask Response obj rather than use jsonify # to take advantage of ujson speed for large str response = make_response(json.dumps({"message": "success", "data": r})) response.headers['Content-Type'] = 'application/json' response.status_code = 200 return response
def indicators(): """ GET/POST the Indicators Controller :return: { 'message': '{success|failure}', 'data': [] } """ if request.method == 'GET': filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) try: if app.config.get('dummy'): r = DummyClient(remote, pull_token()).filter( filters=filters, limit=request.args.get('limit')) else: r = Client(remote, pull_token()).filter( filters=filters, limit=request.args.get('limit')) except RuntimeError as e: logger.error(e) response = jsonify({"message": "search failed", "data": []}) response.status_code = 403 except InvalidSearch as e: logger.error(e) response = jsonify({"message": "invalid search", "data": []}) response.status_code = 400 else: response = jsonify({"message": "success", "data": r}) if request.args.get('gzip'): response.data = compress(response.data) response.status_code = 200 else: fireball = False if request.headers.get('Content-Length'): logger.debug('content-length: %s' % request.headers['Content-Length']) if int(request.headers['Content-Length']) > 10000: logger.info('fireball mode') fireball = True try: data = request.data.decode('utf-8') r = Client(remote, pull_token()).indicators_create(data, fireball=fireball) except RuntimeError as e: logger.error(e) response = jsonify({"message": "submission failed", "data": []}) response.status_code = 422 except TimeoutError as e: logger.error(e) response = jsonify({"message": "submission failed", "data": []}) response.status_code = 408 except Exception as e: logger.error(e) response = jsonify({"message": "submission failed", "data": []}) response.status_code = 422 else: response = jsonify({"message": "success", "data": r}) response.status_code = 201 return response
def feed(): filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) DAYS = request.args.get('days', 30) LIMIT = request.args.get('limit', 50000) now = arrow.utcnow() filters['reporttimeend'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) now = now.replace(days=-DAYS) filters['reporttime'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) try: if app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters) else: r = Client(remote, pull_token()).indicators_search(filters) except RuntimeError as e: logger.error(e) response = jsonify({"message": "search failed", "data": []}) response.status_code = 403 except InvalidSearch as e: logger.error(e) response = jsonify({"message": "invalid search", "data": []}) response.status_code = 400 else: r = aggregate(r) wl_filters = copy.deepcopy(filters) wl_filters['tags'] = 'whitelist' wl_filters['confidence'] = 25 now = arrow.utcnow() now = now.replace(days=-DAYS) wl_filters['reporttime'] = '{}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) wl_filters['nolog'] = True wl_filters['limit'] = 25000 if app.config.get('dummy'): wl = DummyClient(remote, pull_token()).indicators_search(wl_filters) else: wl = Client(remote, pull_token()).indicators_search(wl_filters) wl = aggregate(wl) from .feed import factory as feed_factory f = feed_factory(filters['itype']) r = f().process(r, wl) response = jsonify({"message": "success", "data": r}) if response_compress(): response.data = compress(response.data) response.status_code = 200 return response
def feed(): filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if len(filters) == 0: response = jsonify({ "message": "invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)", "data": [] }) response.status_code = 400 return response # test to make sure feed type exists feed_type = feed_factory(filters['itype']) if feed_type is None: response = jsonify({ "message": "invalid feed itype: {}, valid types are [{}]".format( filters['itype'], '|'.join(FEED_PLUGINS)), "data": [] }) response.status_code = 400 return response DAYS = request.args.get('days', 30) LIMIT = request.args.get('limit', 50000) now = arrow.utcnow() filters['reporttimeend'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) now = now.replace(days=-DAYS) filters['reporttime'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) try: if app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters) else: r = Client(remote, pull_token()).indicators_search(filters) except RuntimeError as e: logger.error(e) response = jsonify({"message": "search failed", "data": []}) response.status_code = 403 except InvalidSearch as e: logger.error(e) response = jsonify({"message": "invalid search", "data": []}) response.status_code = 400 else: r = aggregate(r) wl_filters = copy.deepcopy(filters) wl_filters['tags'] = 'whitelist' wl_filters['confidence'] = 25 now = arrow.utcnow() now = now.replace(days=-DAYS) wl_filters['reporttime'] = '{}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) wl_filters['nolog'] = True wl_filters['limit'] = 25000 if app.config.get('dummy'): wl = DummyClient(remote, pull_token()).indicators_search(wl_filters) else: wl = Client(remote, pull_token()).indicators_search(wl_filters) wl = aggregate(wl) f = feed_factory(filters['itype']) r = f().process(r, wl) response = jsonify({"message": "success", "data": r}) if response_compress(): response.data = compress(response.data) response.status_code = 200 return response