def handle_indicators_create(self, token, data, id=None, client_id=None, flush=False): # this will raise AuthError if false t = self.store.tokens.write(token) if len(data) > 1: start_time = time.time() logger.info('attempting to insert %d indicators..', len(data)) # this will raise AuthError if the groups don't match if isinstance(data, dict): data = [data] for i in data: if not i.get('group'): raise InvalidIndicator('missing group') if i['group'] not in t['groups']: raise AuthError('unable to write to %s' % i['group']) if i.get('message'): try: i['message'] = str(b64decode(data['message'])) except (TypeError, binascii.Error) as e: pass n = self.store.indicators.upsert(t, data, flush=flush) #n = len(data) t = time.time() - start_time logger.info( 'actually inserted %d indicators.. took %0.2f seconds (%0.2f/sec)', n, t, (n / t)) return n data = data[0] if data['group'] not in t['groups']: raise AuthError('unauthorized to write to group: %s' % g) if data.get('message'): try: data['message'] = str(b64decode(data['message'])) except (TypeError, binascii.Error) as e: pass if not self.create_queue.get(token): self.create_queue[token] = {'count': 0, "messages": []} self.create_queue[token]['count'] += 1 self.create_queue_count += 1 self.create_queue[token]['last_activity'] = time.time() self.create_queue[token]['messages'].append((id, client_id, [data])) return MORE_DATA_NEEDED
def handle_indicators_create(self, token, data, id=None, client_id=None): if len(data) == 1: if not self.store.token_write(token): raise AuthError('invalid token') if not self.create_queue.get(token): self.create_queue[token] = {'count': 0, "messages": []} self.create_queue[token]['count'] += 1 self.create_queue_count += 1 self.create_queue[token]['last_activity'] = time.time() if self.create_queue[token]['count'] > self.create_queue_limit: self.create_queue[token]['messages'].append( (id, client_id, data)) return MORE_DATA_NEEDED if self.store.token_write(token): start_time = time.time() if len(data) > 1: logger.info('Upserting %d indicators..', len(data)) r = self.store.indicators_upsert(data) if len(data) > 1: logger.info('Upserting %d indicators.. took %0.2f seconds', len(data), time.time() - start_time) return r else: raise AuthError('invalid token')
def _recv(self, decode=True): mtype, data = Msg().recv(self.socket) if not decode: return data data = json.loads(data) if data.get('message') == 'unauthorized': raise AuthError() if data.get('message') == 'busy': raise CIFBusy() if data.get('message') == 'invalid search': raise InvalidSearch() if data.get('status') != 'success': raise RuntimeError(data.get('message')) try: data['data'] = zlib.decompress(data['data']) except (zlib.error, TypeError): pass return data.get('data')
def create(self, token, data, raw=False, bulk=False): index = self._create_index() expand_ip_idx(data) id = i_to_id(data) if data.get('group') and type(data['group']) != list: data['group'] = [data['group']] if not data.get('lasttime'): data['lasttime'] = arrow.utcnow().datetime.replace(tzinfo=None) if not data.get('firsttime'): data['firsttime'] = data['lasttime'] if not data.get('reporttime'): data['reporttime'] = data['lasttime'] if bulk: d = {'_index': index, '_type': 'indicator', '_source': data} return d data['meta'] = {} data['meta']['index'] = index data['meta']['id'] = id i = Indicator(**data) if not i.save(): raise AuthError('indicator exists') if raw: return i return i.to_dict()
def create(self, token, data, raw=False): index = self._create_index() data['meta'] = {} data['meta']['index'] = index data['meta']['timestamp'] = data.get('lasttime', data['reporttime']) self._check_token_groups(token, data) self._expand_ip_idx(data) if data.get('group') and type(data['group']) != list: data['group'] = [data['group']] if data.get('message'): try: data['message'] = str(b64decode(data['message'])) except (TypeError, binascii.Error) as e: pass i = Indicator(**data) if not i.save(): raise AuthError('invalid token') if raw: return i return i.to_dict()
def handle_indicators_create(self, token, data, id=None, client_id=None): # this will raise AuthError if false t = self.store.tokens.write(token) if len(data) >= 1: start_time = time.time() if len(data) > 1: logger.info('Upserting %d indicators..', len(data)) # this will raise AuthError if the groups don't match r = self.store.indicators.upsert(t, data) if len(data) > 1: logger.info('Upserting %d indicators.. took %0.2f seconds', len(data), time.time() - start_time) return r if data['group'] not in t['groups']: raise AuthError('unauthorized to write to group: %s' % g) if not self.create_queue.get(token): self.create_queue[token] = {'count': 0, "messages": []} self.create_queue[token]['count'] += 1 self.create_queue_count += 1 self.create_queue[token]['last_activity'] = time.time() if self.create_queue[token]['count'] > self.create_queue_limit: self.create_queue[token]['messages'].append((id, client_id, data)) return MORE_DATA_NEEDED
def indicators_create(self, token, data): if self.token_write(token): if type(data) == dict: data = [data] self.logger.debug(data) s = self.handle() for d in data: # namespace conflict with related self.tags tags = d.get("tags", []) if len(tags) > 0: print type(tags) if type(tags) == str or type(tags) == unicode: if '.' in tags: tags = tags.split(',') else: tags = [str(tags)] del d['tags'] o = Indicator(**d) s.add(o) for t in tags: t = Tag(tag=t, indicator=o) s.add(t) s.commit() self.logger.debug('oid: {}'.format(o.id)) return o.id else: raise AuthError('invalid token')
def create(self, token, data, raw=False, bulk=False): index = self._create_index() expand_ip_idx(data) id = i_to_id(data) if data.get('group') and type(data['group']) != list: data['group'] = [data['group']] if bulk: d = {'_index': index, '_type': 'indicator', '_source': data} return d data['meta'] = {} data['meta']['index'] = index data['meta']['id'] = id i = Indicator(**data) if not i.save(): raise AuthError('indicator exists') if raw: return i return i.to_dict()
def _check_status(self, resp, expect=200): if resp.status_code == 400: r = json.loads(resp.text) raise InvalidSearch(r['message']) if resp.status_code == 401: raise AuthError('unauthorized') if resp.status_code == 404: raise NotFound('not found') if resp.status_code == 408: raise TimeoutError('timeout') if resp.status_code == 422: msg = json.loads(resp.text) raise SubmissionFailed(msg['message']) if resp.status_code == 429: raise CIFBusy('RateLimit exceeded') if resp.status_code in [500, 501, 502, 503, 504]: raise CIFBusy('system seems busy..') if resp.status_code != expect: msg = 'unknown: %s' % resp.content raise RuntimeError(msg)
def indicators_create(self, token, data): index = self._create_index() self.logger.debug('index: {}'.format(index)) data['meta'] = {} data['meta']['index'] = index if resolve_itype(data['indicator']) == 'ipv4': import re match = re.search('^(\S+)\/(\d+)$', data['indicator']) if match: data['indicator_ipv4'] = match.group(1) data['indicator_ipv4_mask'] = match.group(2) else: data['indicator_ipv4'] = data['indicator'] if type(data['group']) != list: data['group'] = [data['group']] self.logger.debug(data) i = Indicator(**data) self.logger.debug(i) if i.save(): return i.__dict__['_d_'] else: raise AuthError('invalid token')
def handle_indicators_search(self, token, data, **kwargs): t = self.store.token_read(token) if not t: raise AuthError('invalid token') if PYVERSION == 2: if data.get('indicator'): if isinstance(data['indicator'], str): data['indicator'] = unicode(data['indicator']) try: x = self.store.indicators_search(data) except Exception as e: logger.error(e) if logger.getEffectiveLevel() == logging.DEBUG: import traceback logger.error(traceback.print_exc()) raise InvalidSearch('invalid search') t = self.store.tokens_search({'token': token}) self._log_search(t, data) if isinstance(x, GeneratorType): x = list(x) for xx in x: if xx.get('message'): xx['message'] = b64encode(xx['message']).encode('utf-8') return x
def _check_indicator(self, i, t): for e in REQUIRED_ATTRIBUTES: if not i.get(e): raise ValueError('missing %s' % e) if i['group'] not in t['groups']: raise AuthError('unable to write to %s' % i['group']) return True
def check(self, token, k, v=True): self._flush_cache() if token in self._cache and self._cache[token].get(k): return self._cache[token] rv = list(self.search({'token': token, k: v})) if len(rv) == 0: raise AuthError('unauthorized') self._cache[token] = rv[0] return rv[0]
def handle_indicators_search(self, token, data): if self.store.token_read(token): self.logger.debug('searching') try: x = self.store.indicators_search(token, data) except Exception as e: self.logger.error(e) raise InvalidSearch('invalid search') else: return x else: raise AuthError('invalid token')
def _check_status(self, resp, expect=200): if resp.status_code == 401: raise AuthError() if resp.status_code == 404: raise NotFound() if resp.status_code == 408: raise TimeoutError() if resp.status_code == 422: raise SubmissionFailed() if resp.status_code != expect: raise RuntimeError(resp.content)
def _get(self, uri, params={}): if not uri.startswith('http'): uri = self.remote + uri body = self.session.get(uri, params=params, verify=self.verify_ssl) if body.status_code > 303: err = 'request failed: %s' % str(body.status_code) logger.error(err) if body.status_code == 401: raise AuthError('invalid token') elif body.status_code == 404: err = 'not found' raise RuntimeError(err) elif body.status_code == 408: raise TimeoutError('timeout') else: try: err = json.loads(body.content).get('message') raise RuntimeError(err) except ValueError as e: err = body.content logger.error(err) raise RuntimeError(err) data = body.content try: data = zlib.decompress(b64decode(data)) except (TypeError, binascii.Error) as e: pass except Exception as e: pass msgs = json.loads(data.decode('utf-8')) if not msgs.get('data'): return msgs if isinstance(msgs['data'], list): for m in msgs['data']: if m.get('message'): try: m['message'] = b64decode(m['message']) except Exception as e: pass return msgs
def indicators_create(self, data, raw=False): index = _create_index() self.logger.debug('index: {}'.format(index)) data['meta'] = {} data['meta']['index'] = index itype = resolve_itype(data['indicator']) if itype is 'ipv4': match = re.search('^(\S+)\/(\d+)$', data['indicator']) if match: data['indicator_ipv4'] = match.group(1) data['indicator_ipv4_mask'] = match.group(2) else: data['indicator_ipv4'] = data['indicator'] elif itype is 'ipv6': match = re.search('^(\S+)\/(\d+)$', data['indicator']) if match: data['indicator_ipv6'] = binascii.b2a_hex( socket.inet_pton(socket.AF_INET6, match.group(1))).decode('utf-8') data['indicator_ipv6_mask'] = match.group(2) else: data['indicator_ipv6'] = binascii.b2a_hex( socket.inet_pton(socket.AF_INET6, data['indicator'])).decode('utf-8') if type(data['group']) != list: data['group'] = [data['group']] if data.get('message'): try: data['message'] = str(b64decode(data['message'])) except (TypeError, binascii.Error) as e: pass i = Indicator(**data) if i.save(): if raw: return i else: return i.__dict__['_d_'] else: raise AuthError('invalid token')
def _recv(self, decode=True, close=True): mtype, data = Msg().recv(self.socket) if close: self.socket.close() if not decode: return data data = json.loads(data) if data.get('message') == 'unauthorized': raise AuthError() if data.get('message') == 'busy': raise CIFBusy() if data.get('message') == 'invalid search': raise InvalidSearch() if data.get('status') != 'success': raise RuntimeError(data.get('message')) if data.get('data') is None: raise RuntimeError('invalid response') if isinstance(data.get('data'), bool): return data['data'] # is this a straight up elasticsearch string? if data['data'] == '{}': return [] if isinstance(data['data'], basestring) and data['data'].startswith('{"hits":{"hits":[{"_source":'): data['data'] = json.loads(data['data']) data['data'] = [r['_source'] for r in data['data']['hits']['hits']] try: data['data'] = zlib.decompress(data['data']) except (zlib.error, TypeError): pass return data.get('data')
def _check_indicator(i, t): if not i.get('group'): i['group'] = t['groups'][0] if not i.get('provider'): i['provider'] = t['username'] if not i.get('tags'): i['tags'] = 'suspicious' if not i.get('reported_at'): i['reported_at'] = arrow.utcnow().datetime for e in REQUIRED_ATTRIBUTES: if not i.get(e): raise ValueError('missing %s' % e) if i['group'] not in t['groups']: raise AuthError('unable to write to %s' % i['group']) return True
def check_token_perms(self, mtype, token, data): """Check token perms/groups and raise AuthError if issues :param mtype(str) :param token(list) - from self.auth.handle_token_search :param data(str) - a str dumped dict from Msg() :return token(dict) - matched token dict """ if not token: raise AuthError('Auth: invalid token provided to API') # if more than one token comes back, shenanigans elif len(token) > 1: raise AuthError('multiple token matches during auth. possible wildcard attempt?') else: token = token[0] # check action (mtype) against token perms if mtype.startswith('tokens') or mtype.endswith('delete'): if not token.get('admin'): raise AuthError('Auth: admin function attempted but supplied token had no admin permission') elif mtype.endswith('create') or mtype.endswith('write'): if not token.get('write'): raise AuthError('Auth: write function attempted but supplied token had no write permission') elif mtype == 'indicators_search': if not token.get('read'): raise AuthError('Auth: read function attempt but supplied token had no read permission') # check action (mtype) against token groups if mtype == 'indicators_create': data = json.loads(data) if isinstance(data, dict): data = [data] for i in data: if i.get('group', 'everyone') not in token['groups']: raise AuthError('Auth: indicator function attempt {} but supplied indicator group {} did not match user token groups {}'.format(mtype, i.get('group', 'everyone'), token['groups'])) # for indicators_search and _delete, they use plural "groups" return token
def handle_indicators_create(self, token, data): if self.store.token_write(token): return self.store.indicators_create(token, data) else: raise AuthError('invalid token')
def handle_indicators_search(self, token, data): if self.store.token_read(token): self.logger.debug('searching') return self.store.indicators_search(token, data) else: raise AuthError('invalid token')
def handle_tokens_edit(self, token, data, **kwargs): if self.store.tokens.admin(token): return self.store.tokens.edit(data) raise AuthError('invalid token')
def handle_tokens_create(self, token, data, **kwargs): if self.store.tokens.admin(token): logger.debug('tokens_create') return self.store.tokens.create(data) raise AuthError('invalid token')
def handle_indicators_create(self, token, data, id=None, client_id=None, flush=False): # this will raise AuthError if false t = self.store.tokens.write(token) if len(data) > 1 or t['username'] == 'admin': start_time = time.time() logger.info('attempting to insert %d indicators..', len(data)) # this will raise AuthError if the groups don't match if isinstance(data, dict): data = [data] for i in data: if not i.get('group'): i['group'] = 'everyone' # optionally force provider to match username with exceptions if i.get('provider') and STRICT_PROVIDERS and not t[ 'username'] in STRICT_PROVIDERS_EXCEPTIONS: i['provider'] = t['username'] if not i.get('provider') or i['provider'] == '': i['provider'] = t['username'] if i['group'] not in t['groups']: raise AuthError('unable to write to %s' % i['group']) if not i.get('tags'): i['tags'] = ['suspicious'] if i.get('message'): try: i['message'] = str(b64decode(data['message'])) except (TypeError, binascii.Error) as e: pass self._timestamps_fix(i) n = self.store.indicators.upsert(t, data, flush=flush) t = time.time() - start_time logger.info( 'actually inserted %d indicators.. took %0.2f seconds (%0.2f/sec)', n, t, (n / t)) return n data = data[0] if data['group'] not in t['groups']: raise AuthError('unauthorized to write to group: %s' % data['group']) if data.get('message'): try: data['message'] = str(b64decode(data['message'])) except (TypeError, binascii.Error) as e: pass if not self.create_queue.get(token): self.create_queue[token] = {'count': 0, "messages": []} self.create_queue[token]['count'] += 1 self.create_queue_count += 1 self.create_queue[token]['last_activity'] = time.time() self.create_queue[token]['messages'].append((id, client_id, [data])) return MORE_DATA_NEEDED
def _flush_create_queue(self): for t in self.create_queue: if len(self.create_queue[t]['messages']) == 0: continue logger.debug('flushing queue...') data = [msg[0] for _, _, msg in self.create_queue[t]['messages']] _t = self.store.tokens.write(t) try: start_time = time.time() logger.info('attempting to insert %d indicators..', len(data)) # this will raise AuthError if the groups don't match if isinstance(data, dict): data = [data] for i in data: if not i.get('group'): i['group'] = 'everyone' # optionally force provider to match username with exceptions if i.get('provider') and STRICT_PROVIDERS and not _t[ 'username'] in STRICT_PROVIDERS_EXCEPTIONS: i['provider'] = _t['username'] if not i.get('provider') or i['provider'] == '': i['provider'] = _t['username'] if i['group'] not in _t['groups']: raise AuthError('unable to write to %s' % i['group']) if not i.get('tags'): i['tags'] = ['suspicious'] if i.get('message'): try: i['message'] = str(b64decode(data['message'])) except (TypeError, binascii.Error) as e: pass self._timestamps_fix(i) n = self.store.indicators.upsert(_t, data) t_time = time.time() - start_time logger.info( 'actually inserted %d indicators.. took %0.2f seconds (%0.2f/sec)', n, t_time, (n / t_time)) if n == 0: rv = {'status': 'failed', 'message': 'invalid indicator'} else: rv = {"status": "success", "data": n} except AuthError as e: rv = {'status': 'failed', 'message': 'unauthorized'} except StoreLockError: rv = {'status': 'failed', 'message': 'busy'} for id, client_id, _ in self.create_queue[t]['messages']: Msg(id=id, client_id=client_id, mtype=Msg.INDICATORS_CREATE, data=rv) if rv['status'] == 'success': self.store.tokens.update_last_activity_at( t, arrow.utcnow().datetime) logger.debug('queue flushed..')
def handle_tokens_delete(self, token, data, **kwargs): if self.store.token_admin(token): return self.store.tokens_delete(data) else: raise AuthError('invalid token')
def handle_tokens_create(self, token, data): if self.store.token_admin(token): self.logger.debug('tokens_create') return self.store.tokens_create(data) else: raise AuthError('invalid token')
def handle_tokens_edit(self, token, data): if self.store.token_admin(token): return self.store.token_edit(data) else: raise AuthError('invalid token')
def handle_indicators_search(self, token, data, **kwargs): if PYVERSION == 2: if data.get('indicator'): if isinstance(data['indicator'], str): data['indicator'] = unicode(data['indicator']) # token acl check if token.get('acl') and token.get('acl') != ['']: if data.get('itype') and data.get('itype') not in token['acl']: raise AuthError('unauthorized to access itype {}'.format( data['itype'])) if not data.get('itype'): data['itype'] = token['acl'] # verify group filter matches token permissions if data.get('groups') and (not token.get('admin') or token.get('admin') == ''): if isinstance(data['groups'], basestring): q_groups = [g.strip() for g in data['groups'].split(',')] elif isinstance(data['groups'], list): q_groups = data['groups'] gg = [] for g in q_groups: if AUTH_ENABLED: if g in token['groups']: gg.append(g) else: gg.append(g) if gg: data['groups'] = gg else: data['groups'] = '{}' if not data.get('reporttime'): if data.get('days'): now = arrow.utcnow() data['reporttimeend'] = '{0}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) now = now.shift(days=-int(data['days'])) data['reporttime'] = '{0}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) if data.get('hours'): now = arrow.utcnow() data['reporttimeend'] = '{0}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) now = now.shift(hours=-int(data['hours'])) data['reporttime'] = '{0}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) s = time.time() self._log_search(token, data) try: x = self.store.indicators.search(token, data) logger.debug('done') except Exception as e: logger.error(e) if logger.getEffectiveLevel() == logging.DEBUG: logger.error(traceback.print_exc()) raise InvalidSearch(': {}'.format(e)) logger.debug('%s' % (time.time() - s)) # for xx in x: # if xx.get('message'): # xx['message'] = b64encode(xx['message']).encode('utf-8') return x