def __init__(self, context, remote=HUNTER_ADDR, router=ROUTER_ADDR, token=None): self.logger = logging.getLogger(__name__) self.context = context self.socket = self.context.socket(zmq.PULL) self.plugins = self._load_plugins() self.hunters = remote # TODO - convert this to an async socket self.router = Client(remote=router, token=token)
def patch(self): cli = Client(remote, pull_token()) try: r = cli.tokens_edit(request.data) except AuthError: return jsonify_unauth() except Exception as e: logger.error(e) return jsonify_unknown() if not r: return jsonify_unauth() return jsonify_success(r)
def post(self): fireball = False nowait = request.args.get('nowait', False) if request.headers.get('Content-Length'): logger.debug('content-length: %s' % request.headers['Content-Length']) if int(request.headers['Content-Length']) > 5000: logger.info('fireball mode') fireball = True try: r = Client(remote, pull_token()).indicators_create(request.data, nowait=nowait, fireball=fireball) if nowait: r = 'pending' except SubmissionFailed as e: logger.error(e) return jsonify_unknown(msg='submission failed: %s' % e, code=422) except RuntimeError as e: logger.error(e) return jsonify_unknown( msg='submission failed, check logs for more information', code=422) except TimeoutError as e: logger.error(e) return jsonify_unknown( 'submission failed, check logs for more information', 408) except CIFBusy: return jsonify_busy() except Exception as e: logger.error(e) return jsonify_unknown( 'submission failed, check logs for more information', 422) except AuthError: return jsonify_unauth() return jsonify_success(r, code=201)
def delete(self, token_id): if not session['admin']: return redirect('/u/login', code=401) filters = {} if token_id: filters['token'] = token_id filters['username'] = None filters = json.dumps(filters) try: r = Client(remote, HTTPD_TOKEN).tokens_delete(filters) except Exception as e: logger.error(e) flash(e, 'error') else: flash('success', 'success') return redirect(url_for('/u/tokens'))
def _search_bulk(f): try: with Client() as client: r = client.indicators_search(f) except zmq.error.Again as e: return api.abort(503) except Exception as e: logger.error(e) if logger.getEffectiveLevel() == logging.DEBUG: traceback.print_exc() if 'invalid search' in str(e): logger.error(e) return api.abort(400, str(e)) return api.abort(500) return r
def get(self): cli = Client(remote, pull_token()) filters = {} for f in TOKEN_FILTERS: filters[f] = request.args.get(f) if current_app.config.get('dummy'): return jsonify_success() try: r = cli.tokens_search(filters) except AuthError: return jsonify_unauth() except Exception as e: logger.error(e) return jsonify_unknown() return jsonify_success(r)
def get(self): filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if request.args.get('q'): filters['indicator'] = request.args.get('q') if current_app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters) return jsonify_success(r) try: r = Client(remote, pull_token()).indicators_search(filters, decode=False) except RuntimeError as e: logger.error(e) return jsonify_unknown(msg='search failed') except InvalidSearch as e: return jsonify_unknown(msg='invalid search', code=400) except AuthError: return jsonify_unauth() except Exception as e: logger.error(e) return jsonify_unknown( msg='search failed, system may be too busy, check back later') response = current_app.response_class(r, mimetype='application/json') if isinstance(r, basestring): if '"message":"unauthorized"' in r and '"message":"unauthorized"' in r: response.status_code = 401 return response return response
def search(): """ Search controller :param str q: query term (ex: example.org, 1.2.3.4, 1.2.3.0/24) :param dict filters: query filters :param int limit: limit search results (reporttime desc) :return: { 'message': 'success', 'data': [] } """ remote = ROUTER_ADDR if app.config.get('CIF_ROUTER_ADDR'): remote = app.config['CIF_ROUTER_ADDR'] filters = {} for k in VALID_FILTERS: if request.args.get(k): filters[k] = request.args.get(k) if request.args.get('q'): filters['indicator'] = request.args.get('q') try: if app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters) else: r = Client(remote, pull_token()).indicators_search(filters) response = jsonify({"message": "success", "data": r}) response.status_code = 200 except AuthError as e: response = jsonify({ 'message': 'unauthorized', 'data': [], 'status': 'failed' }) response.status_code = 401 return response
def delete(self): try: data = request.data.decode('utf-8') with Client(remote, pull_token()) as cli: r = cli.indicators_delete(data) except RuntimeError as e: logger.error(e) return jsonify_unknown(msg='submission failed, check logs for more information', code=422) except TimeoutError as e: logger.error(e) return jsonify_unknown('submission failed, check logs for more information', 408) except Exception as e: logger.error(e) return jsonify_unknown('submission failed, check logs for more information', 422) except AuthError: return jsonify_unauth() return jsonify_success(r)
def post(self): cli = Client(remote, pull_token()) if not request.data: return jsonify_unknown('missing data', 400) if current_app.config.get('dummy'): return jsonify_success() try: r = cli.tokens_create(request.data) except AuthError: return jsonify_unauth() except Exception as e: logger.error(e) return jsonify_unknown() if not r: return jsonify_unknown('create failed', 400) return jsonify_success(r, code=201)
def login(): if request.method == 'GET': return render_template('login.html') if request.method == 'POST': from cifsdk.client.zeromq import ZMQ as Client if request.form['token'] == '': return render_template('login.html') c = Client(remote, HTTPD_TOKEN) rv = c.tokens_search({'token': request.form['token']}) if len(rv) == 0: return render_template('login.html', code=401) user = rv[0] if user['revoked']: return render_template('login.html', code=401) for e in ['username', 'token', 'admin', 'read', 'write', 'groups']: session[e] = user[e] return redirect(url_for('/u/search'))
def get(self): remote = ROUTER_ADDR if current_app.config.get('CIF_ROUTER_ADDR'): remote = current_app.config['CIF_ROUTER_ADDR'] write = request.args.get('write', None) if current_app.config.get('dummy'): r = DummyClient(remote, pull_token()).ping(write=write) return jsonify_success(r) try: r = Client(remote, pull_token()).ping(write=write) except TimeoutError: return jsonify_unknown(msg='timeout', code=408) except AuthError: return jsonify_unauth() if not r: return jsonify_unknown(503) return jsonify_success(r)
def start(self): # TODO - convert this to an async socket router = Client(remote=self.router, token=self.token, nowait=True) plugins = self._load_plugins() socket = zmq.Context().socket(zmq.PULL) socket.SNDTIMEO = SNDTIMEO socket.set_hwm(ZMQ_HWM) logger.debug('connecting to {}'.format(self.hunters)) socket.connect(self.hunters) logger.debug('starting hunter') try: while True: logger.debug('waiting...') data = socket.recv() logger.debug(data) data = json.loads(data) if isinstance(data, dict): data = [data] for d in data: d = Indicator(**d) for p in plugins: try: p.process(d, router) except Exception as e: logger.error(e) traceback.print_exc() logger.error('giving up on: {}'.format(d)) except KeyboardInterrupt: logger.info('shutting down hunter...') return
def DataTables(): filters = {} if session['filters'].get('q'): q = session['filters'].get('q') if q in ['ipv4', 'ipv6', 'fqdn', 'url', 'email']: filters['itype'] = q else: filters['indicator'] = q if session['filters'].get('confidence'): filters['confidence'] = session['filters'].get('confidence') if session['filters'].get('provider'): filters['provider'] = session['filters'].get('provider') if session['filters'].get('group'): filters['groups'] = session['filters'].get('group') if session['filters'].get('tags'): filters['tags'] = session['filters'].get('tags') if session['filters'].get('reporttime'): filters['reporttime'] = session['filters'].get('reporttime') if not session['filters']: return [] try: r = Client(remote, session['token']).indicators_search(filters) except Exception as e: logger.error(e) flash(e, 'error') response = [] else: response = r finally: session['filters'] = {} return response
def start(self): router = Client(remote=self.router, token=self.token, nowait=True) plugins = self._load_plugins() socket = zmq.Context().socket(zmq.PULL) socket.SNDTIMEO = SNDTIMEO socket.set_hwm(ZMQ_HWM) logger.debug('connecting to {}'.format(self.hunters)) socket.connect(self.hunters) logger.debug('starting hunter') poller = zmq.Poller() poller.register(socket, zmq.POLLIN) while not self.exit.is_set(): try: s = dict(poller.poll(1000)) except SystemExit or KeyboardInterrupt: break if socket not in s: continue data = socket.recv_multipart() logger.debug(data) data = json.loads(data[0]) if isinstance(data, dict): if not data.get('indicator'): continue if not data.get('itype'): data = Indicator( indicator=data['indicator'], tags='search', confidence=10, group='everyone', tlp='amber', ).__dict__() if not data.get('tags'): data['tags'] = [] data = [data] for d in data: d = Indicator(**d) if d.indicator in ["", 'localhost', 'example.com']: continue if self.exclude.get(d.provider): for t in d.tags: if t in self.exclude[d.provider]: logger.debug('skipping: {}'.format(d.indicator)) for p in plugins: if p.is_advanced: if not HUNTER_ADVANCED: continue try: p.process(d, router) except Exception as e: logger.error(e) logger.error('[{}] giving up on: {}'.format(p, d))
def start(self): router = Client(remote=self.router, token=self.token, nowait=True, autoclose=False) plugins = self._load_plugins() socket = zmq.Context().socket(zmq.PULL) socket.SNDTIMEO = SNDTIMEO socket.set_hwm(ZMQ_HWM) logger.debug('connecting to {}'.format(self.hunters)) socket.connect(self.hunters) logger.debug('starting hunter') poller = zmq.Poller() poller.register(socket, zmq.POLLIN) while not self.exit.is_set(): try: s = dict(poller.poll(1000)) except SystemExit or KeyboardInterrupt: break if socket not in s: continue id, token, mtype, data = Msg().recv(socket) data = json.loads(data) if isinstance(data, dict): if not data.get('indicator'): continue if not data.get('itype'): try: data = Indicator( indicator=data['indicator'], tags='search', confidence=10, group='everyone', tlp='amber', ).__dict__() except InvalidIndicator: logger.debug('skipping invalid indicator: {}'.format( data['indicator'])) continue if not data.get('tags'): data['tags'] = [] data = [data] token = json.loads(token) for d in data: d = Indicator(**d) if d.confidence < HUNTER_MIN_CONFIDENCE: continue # prevent hunter recursion if disabled if not HUNTER_RECURSION and d.tags and 'hunter' in d.tags: continue if d.indicator in ["", 'localhost', 'example.com']: continue if self.exclude.get(d.provider): for t in d.tags: if t in self.exclude[d.provider]: logger.debug('skipping: {}'.format(d.indicator)) continue for p in plugins: if p.is_advanced: if not HUNTER_ADVANCED: continue try: p.process(i=d, router=router, user_token=token) except Exception as e: logger.error(e) logger.error('[{}] giving up on: {}'.format(p, d))
def get(self): filters = {} start = time.time() id = g.sid for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if len(filters) == 0: return jsonify_unknown( 'invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)', 400) if 'itype' not in filters: return jsonify_unknown( 'missing itype filter (ipv4, fqdn, url, etc...)', 400) # test to make sure feed type exists feed_type = feed_factory(filters['itype']) if feed_type is None: err = "invalid feed itype: {}, valid types are [{}]".format( filters['itype'], '|'.join(FEED_PLUGINS)) return jsonify_unknown(err, 400) if not filters.get('reporttime'): if not filters.get('days'): if not filters.get('itype'): filters['days'] = DAYS_SHORT else: filters['days'] = FEED_DAYS[filters['itype']] if not filters.get('limit'): filters['limit'] = FEEDS_LIMIT if current_app.config.get('dummy'): if current_app.config.get('feed'): r = DummyClient(remote, pull_token()).indicators_search( filters, decode=True, test_data=current_app.config['feed']['data'], test_wl=current_app.config['feed']['wl']) else: r = DummyClient(remote, pull_token()).indicators_search(filters) return jsonify_success(r) else: logger.debug('%s building feed' % id) logger.debug('%s getting dataset' % id) try: r = Client(remote, pull_token()).indicators_search(filters) except AuthError: return jsonify_unauth() except RuntimeError as e: return jsonify_unknown('search failed', 403) except InvalidSearch as e: logger.error(e) return jsonify_unknown('invalid search', 400) except Exception as e: logger.error(e) return jsonify_unknown(msg='search failed') r = aggregate(r) wl_filters = copy.deepcopy(filters) # whitelists are typically updated 1/month so we should catch those # esp for IP addresses if not wl_filters.get('days') or int(wl_filters['days']) < 45: wl_filters['days'] = 45 if wl_filters.get('reporttime'): del wl_filters['reporttime'] wl_filters['tags'] = 'whitelist' wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE wl_filters['nolog'] = True wl_filters['limit'] = FEEDS_WHITELIST_LIMIT logger.debug('gathering whitelist..') if current_app.config.get('feed') and current_app.config.get( 'feed').get('wl'): wl = current_app.config.get('feed').get('wl') else: try: wl = Client(remote, pull_token()).indicators_search(wl_filters) except Exception as e: logger.error(e) return jsonify_unknown('feed query failed', 503) logger.debug('%s aggregating' % id) wl = aggregate(wl) f = feed_factory(filters['itype']) logger.debug('%s merging' % id) r = f().process(r, wl) logger.debug('%s done: %s' % (id, str((time.time() - start)))) response = jsonify({"message": "success", "data": r}) response.status_code = 200 return response
def client(): yield Client()
def tokens(): cli = Client(remote, pull_token()) if request.method == 'DELETE': try: r = cli.tokens_delete(request.data) except Exception as e: logger.error(e) response = jsonify({"message": "failed", "data": []}) response.status_code = 503 else: response = jsonify({'message': 'success', 'data': r}) response.status_code = 200 elif request.method == 'POST': if request.data: try: r = cli.tokens_create(request.data) except AuthError: response = jsonify({ 'message': 'admin privs required', 'data': [] }) response.status_code = 401 except Exception as e: logger.error(e) response = jsonify({'message': 'create failed', 'data': []}) response.status_code = 503 else: if r: response = jsonify({'message': 'success', 'data': r}) response.status_code = 200 else: response = jsonify({ 'message': 'admin privs required', 'data': [] }) response.status_code = 401 else: response = jsonify({'message': 'create failed', 'data': []}) response.status_code = 400 elif request.method == 'PATCH': try: r = cli.tokens_edit(request.data) except AuthError: response = jsonify({'message': 'admin privs required', 'data': []}) response.status_code = 401 except Exception as e: logger.error(e) import traceback traceback.print_exc() response = jsonify({'message': 'create failed', 'data': []}) response.status_code = 503 else: if r: response = jsonify({'message': 'success', 'data': r}) response.status_code = 200 else: response = jsonify({ 'message': 'admin privs required', 'data': [] }) response.status_code = 401 else: filters = {} for f in TOKEN_FILTERS: filters[f] = request.args.get(f) try: r = cli.tokens_search(filters) except AuthError: response = jsonify({"message": "failed", "data": []}) response.status_code = 401 except Exception as e: logger.error(e) response = jsonify({"message": "failed", "data": []}) response.status_code = 503 else: response = jsonify({'message': 'success', 'data': r}) response.status_code = 200 return response
def get(self): filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if len(filters) == 0: return jsonify_unknown('invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)', 400) if 'itype' not in filters: return jsonify_unknown('missing itype filter (ipv4, fqdn, url, etc...)', 400) # test to make sure feed type exists feed_type = feed_factory(filters['itype']) if feed_type is None: err = "invalid feed itype: {}, valid types are [{}]".format(filters['itype'], '|'.join(FEED_PLUGINS)) return jsonify_unknown(err, 400) if not filters.get('days'): filters['days'] = FEEDS_DAYS if not filters.get('limit'): filters['limit'] = FEEDS_LIMIT if current_app.config.get('dummy'): if current_app.config.get('feed'): r = DummyClient(remote, pull_token()).indicators_search(filters, decode=True, test_data=current_app.config['feed']['data'], test_wl=current_app.config['feed']['wl']) else: r = DummyClient(remote, pull_token()).indicators_search(filters) return jsonify_success(r) else: try: r = Client(remote, pull_token()).indicators_search(filters) except AuthError: return jsonify_unauth() except RuntimeError as e: return jsonify_unknown('search failed', 403) except InvalidSearch as e: logger.error(e) return jsonify_unknown('invalid search', 400) except Exception as e: logger.error(e) return jsonify_unknown(msg='search failed') r = aggregate(r) wl_filters = copy.deepcopy(filters) wl_filters['tags'] = 'whitelist' wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE wl_filters['nolog'] = True wl_filters['limit'] = FEEDS_WHITELIST_LIMIT if current_app.config.get('feed').get('wl'): wl = current_app.config.get('feed').get('wl') else: try: wl = Client(remote, pull_token()).indicators_search(wl_filters) except Exception as e: logger.error(e) return jsonify_unknown('feed query failed', 503) wl = aggregate(wl) f = feed_factory(filters['itype']) r = f().process(r, wl) response = jsonify({ "message": "success", "data": r }) response.status_code = 200 return response
def search(): """ Search controller :param str q: query term (ex: example.org, 1.2.3.4, 1.2.3.0/24) :param dict filters: query filters :param int limit: limit search results (reporttime desc) :return: { 'message': 'success', 'data': [] } """ remote = ROUTER_ADDR if app.config.get('CIF_ROUTER_ADDR'): remote = app.config['CIF_ROUTER_ADDR'] filters = {} for k in VALID_FILTERS: if request.args.get(k): filters[k] = request.args.get(k) if request.args.get('q'): filters['indicator'] = request.args.get('q') try: if request_v2(): if app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters) else: r = Client(remote, pull_token()).indicators_search(filters) for rr in r: rr['observable'] = rr['indicator'] del rr['indicator'] if rr.get('itype'): rr['otype'] = rr['itype'] del rr['itype'] response = jsonify({'message': 'success', 'data': r}) else: if app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters, decode=False) else: r = Client(remote, pull_token()).indicators_search(filters, decode=False) response = current_app.response_class(r, mimetype='application/json') if response_compress(): logger.debug('compressing') response.data = compress(response.data) response.status_code = 200 if isinstance(r, basestring): if '"message":"unauthorized"' in r.decode( 'utf-8') and '"message":"unauthorized"' in r.decode( 'utf-8'): response.status_code = 401 except AuthError as e: response = jsonify({ 'message': 'unauthorized', 'data': [], 'status': 'failed' }) response.status_code = 401 except InvalidSearch as e: logger.error(e) response = jsonify({"message": "invalid search", "data": []}) response.status_code = 400 return response
def feed(): filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if len(filters) == 0: response = jsonify({ "message": "invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)", "data": [] }) response.status_code = 400 return response # test to make sure feed type exists feed_type = feed_factory(filters['itype']) if feed_type is None: response = jsonify({ "message": "invalid feed itype: {}, valid types are [{}]".format( filters['itype'], '|'.join(FEED_PLUGINS)), "data": [] }) response.status_code = 400 return response DAYS = request.args.get('days', 30) LIMIT = request.args.get('limit', 50000) now = arrow.utcnow() filters['reporttimeend'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) now = now.replace(days=-DAYS) filters['reporttime'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) try: if app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters) else: r = Client(remote, pull_token()).indicators_search(filters) except RuntimeError as e: logger.error(e) response = jsonify({"message": "search failed", "data": []}) response.status_code = 403 except InvalidSearch as e: logger.error(e) response = jsonify({"message": "invalid search", "data": []}) response.status_code = 400 else: r = aggregate(r) wl_filters = copy.deepcopy(filters) wl_filters['tags'] = 'whitelist' wl_filters['confidence'] = 25 now = arrow.utcnow() now = now.replace(days=-DAYS) wl_filters['reporttime'] = '{}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) wl_filters['nolog'] = True wl_filters['limit'] = 25000 if app.config.get('dummy'): wl = DummyClient(remote, pull_token()).indicators_search(wl_filters) else: wl = Client(remote, pull_token()).indicators_search(wl_filters) wl = aggregate(wl) f = feed_factory(filters['itype']) r = f().process(r, wl) response = jsonify({"message": "success", "data": r}) if response_compress(): response.data = compress(response.data) response.status_code = 200 return response
def get(self): filters = {} start = time.time() id = g.sid for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) if len(filters) == 0: return jsonify_unknown( 'invalid search, missing an itype filter (ipv4, fqdn, url, sha1...)', 400) if 'itype' not in filters: return jsonify_unknown( 'missing itype filter (ipv4, fqdn, url, etc...)', 400) if filters.get('tags'): if 'whitelist' in filters.get('tags'): return jsonify_unknown( 'Invalid filter: tag "whitelist" is invalid for a feed. To find allow-listed indicators, perform a regular search rather than a feed pull', 400) else: # add a negation to ensure our search doesn't include allowlisted items filters['tags'] += ',!whitelist' else: filters['tags'] = '!whitelist' # test to make sure feed type exists feed_type = feed_factory(filters['itype']) if feed_type is None: err = "invalid feed itype: {}, valid types are [{}]".format( filters['itype'], '|'.join(FEED_PLUGINS)) return jsonify_unknown(err, 400) if not filters.get('reporttime'): if not filters.get('days'): if not filters.get('itype'): filters['days'] = DAYS_SHORT else: filters['days'] = FEED_DAYS[filters['itype']] if not filters.get('limit'): filters['limit'] = FEEDS_LIMIT if current_app.config.get('dummy'): if current_app.config.get('feed'): r = DummyClient(remote, pull_token()).indicators_search( filters, decode=True, test_data=current_app.config['feed']['data'], test_wl=current_app.config['feed']['wl']) else: r = DummyClient(remote, pull_token()).indicators_search(filters) return jsonify_success(r) else: logger.debug('%s building feed' % id) logger.debug('%s getting dataset' % id) try: r = Client(remote, pull_token()).indicators_search(filters) except AuthError: return jsonify_unauth() except RuntimeError as e: return jsonify_unknown('search failed', 403) except InvalidSearch as e: logger.error(e) return jsonify_unknown('invalid search', 400) except Exception as e: logger.error(e) return jsonify_unknown(msg='search failed') r = aggregate(r) wl_filters = copy.deepcopy(filters) # whitelists are typically updated 1/month so we should catch those # esp for IP addresses if not wl_filters.get('days') or int(wl_filters['days']) < 45: wl_filters['days'] = 45 if wl_filters.get('reporttime'): del wl_filters['reporttime'] wl_filters['tags'] = 'whitelist' wl_filters['confidence'] = HTTPD_FEED_WHITELIST_CONFIDENCE wl_filters['nolog'] = True wl_filters['limit'] = FEEDS_WHITELIST_LIMIT # remove provider from wl_filters if exists (we don't want to narrow wl scope by provider) wl_filters.pop('provider', None) logger.debug('gathering whitelist..') if current_app.config.get('feed') and current_app.config.get( 'feed').get('wl'): wl = current_app.config.get('feed').get('wl') else: try: wl = Client(internal_remote, pull_token()).indicators_search(wl_filters) except Exception as e: logger.error(e) return jsonify_unknown('feed query failed', 503) logger.debug('%s aggregating' % id) wl = aggregate(wl) f = feed_factory(filters['itype']) logger.debug('%s merging' % id) r = f().process(r, wl) logger.debug('%s done: %s' % (id, str((time.time() - start)))) # manually make flask Response obj rather than use jsonify # to take advantage of ujson speed for large str response = make_response(json.dumps({"message": "success", "data": r})) response.headers['Content-Type'] = 'application/json' response.status_code = 200 return response
def feed(): filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) DAYS = request.args.get('days', 30) LIMIT = request.args.get('limit', 50000) now = arrow.utcnow() filters['reporttimeend'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) now = now.replace(days=-DAYS) filters['reporttime'] = '{}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) try: if app.config.get('dummy'): r = DummyClient(remote, pull_token()).indicators_search(filters) else: r = Client(remote, pull_token()).indicators_search(filters) except RuntimeError as e: logger.error(e) response = jsonify({"message": "search failed", "data": []}) response.status_code = 403 except InvalidSearch as e: logger.error(e) response = jsonify({"message": "invalid search", "data": []}) response.status_code = 400 else: r = aggregate(r) wl_filters = copy.deepcopy(filters) wl_filters['tags'] = 'whitelist' wl_filters['confidence'] = 25 now = arrow.utcnow() now = now.replace(days=-DAYS) wl_filters['reporttime'] = '{}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) wl_filters['nolog'] = True wl_filters['limit'] = 25000 if app.config.get('dummy'): wl = DummyClient(remote, pull_token()).indicators_search(wl_filters) else: wl = Client(remote, pull_token()).indicators_search(wl_filters) wl = aggregate(wl) from .feed import factory as feed_factory f = feed_factory(filters['itype']) r = f().process(r, wl) response = jsonify({"message": "success", "data": r}) if response_compress(): response.data = compress(response.data) response.status_code = 200 return response
def indicators(): """ GET/POST the Indicators Controller :return: { 'message': '{success|failure}', 'data': [] } """ if request.method == 'GET': filters = {} for f in VALID_FILTERS: if request.args.get(f): filters[f] = request.args.get(f) try: if app.config.get('dummy'): r = DummyClient(remote, pull_token()).filter( filters=filters, limit=request.args.get('limit')) else: r = Client(remote, pull_token()).filter( filters=filters, limit=request.args.get('limit')) except RuntimeError as e: logger.error(e) response = jsonify({"message": "search failed", "data": []}) response.status_code = 403 except InvalidSearch as e: logger.error(e) response = jsonify({"message": "invalid search", "data": []}) response.status_code = 400 else: response = jsonify({"message": "success", "data": r}) if request.args.get('gzip'): response.data = compress(response.data) response.status_code = 200 else: fireball = False if request.headers.get('Content-Length'): logger.debug('content-length: %s' % request.headers['Content-Length']) if int(request.headers['Content-Length']) > 10000: logger.info('fireball mode') fireball = True try: data = request.data.decode('utf-8') r = Client(remote, pull_token()).indicators_create(data, fireball=fireball) except RuntimeError as e: logger.error(e) response = jsonify({"message": "submission failed", "data": []}) response.status_code = 422 except TimeoutError as e: logger.error(e) response = jsonify({"message": "submission failed", "data": []}) response.status_code = 408 except Exception as e: logger.error(e) response = jsonify({"message": "submission failed", "data": []}) response.status_code = 422 else: response = jsonify({"message": "success", "data": r}) response.status_code = 201 return response
def test_client_zmq(): cli = Client(ROUTER_ADDR, '12345') assert cli.remote == ROUTER_ADDR assert cli.token == '12345'