def handle_indicators_create(self,
token,
data,
id=None,
client_id=None,
flush=False):
# this will raise AuthError if false
t = self.store.tokens.write(token)
if len(data) > 1:
start_time = time.time()
logger.info('attempting to insert %d indicators..', len(data))
# this will raise AuthError if the groups don't match
if isinstance(data, dict):
data = [data]
for i in data:
if not i.get('group'):
raise InvalidIndicator('missing group')
if i['group'] not in t['groups']:
raise AuthError('unable to write to %s' % i['group'])
if i.get('message'):
try:
i['message'] = str(b64decode(data['message']))
except (TypeError, binascii.Error) as e:
pass
n = self.store.indicators.upsert(t, data, flush=flush)
#n = len(data)
t = time.time() - start_time
logger.info(
'actually inserted %d indicators.. took %0.2f seconds (%0.2f/sec)',
n, t, (n / t))
return n
data = data[0]
if data['group'] not in t['groups']:
raise AuthError('unauthorized to write to group: %s' % g)
if data.get('message'):
try:
data['message'] = str(b64decode(data['message']))
except (TypeError, binascii.Error) as e:
pass
if not self.create_queue.get(token):
self.create_queue[token] = {'count': 0, "messages": []}
self.create_queue[token]['count'] += 1
self.create_queue_count += 1
self.create_queue[token]['last_activity'] = time.time()
self.create_queue[token]['messages'].append((id, client_id, [data]))
return MORE_DATA_NEEDED
def handle_indicators_create(self, token, data, id=None, client_id=None):
if len(data) == 1:
if not self.store.token_write(token):
raise AuthError('invalid token')
if not self.create_queue.get(token):
self.create_queue[token] = {'count': 0, "messages": []}
self.create_queue[token]['count'] += 1
self.create_queue_count += 1
self.create_queue[token]['last_activity'] = time.time()
if self.create_queue[token]['count'] > self.create_queue_limit:
self.create_queue[token]['messages'].append(
(id, client_id, data))
return MORE_DATA_NEEDED
if self.store.token_write(token):
start_time = time.time()
if len(data) > 1:
logger.info('Upserting %d indicators..', len(data))
r = self.store.indicators_upsert(data)
if len(data) > 1:
logger.info('Upserting %d indicators.. took %0.2f seconds',
len(data),
time.time() - start_time)
return r
else:
raise AuthError('invalid token')
def _recv(self, decode=True):
mtype, data = Msg().recv(self.socket)
if not decode:
return data
data = json.loads(data)
if data.get('message') == 'unauthorized':
raise AuthError()
if data.get('message') == 'busy':
raise CIFBusy()
if data.get('message') == 'invalid search':
raise InvalidSearch()
if data.get('status') != 'success':
raise RuntimeError(data.get('message'))
try:
data['data'] = zlib.decompress(data['data'])
except (zlib.error, TypeError):
pass
return data.get('data')
def create(self, token, data, raw=False, bulk=False):
index = self._create_index()
expand_ip_idx(data)
id = i_to_id(data)
if data.get('group') and type(data['group']) != list:
data['group'] = [data['group']]
if not data.get('lasttime'):
data['lasttime'] = arrow.utcnow().datetime.replace(tzinfo=None)
if not data.get('firsttime'):
data['firsttime'] = data['lasttime']
if not data.get('reporttime'):
data['reporttime'] = data['lasttime']
if bulk:
d = {'_index': index, '_type': 'indicator', '_source': data}
return d
data['meta'] = {}
data['meta']['index'] = index
data['meta']['id'] = id
i = Indicator(**data)
if not i.save():
raise AuthError('indicator exists')
if raw:
return i
return i.to_dict()
def create(self, token, data, raw=False):
index = self._create_index()
data['meta'] = {}
data['meta']['index'] = index
data['meta']['timestamp'] = data.get('lasttime', data['reporttime'])
self._check_token_groups(token, data)
self._expand_ip_idx(data)
if data.get('group') and type(data['group']) != list:
data['group'] = [data['group']]
if data.get('message'):
try:
data['message'] = str(b64decode(data['message']))
except (TypeError, binascii.Error) as e:
pass
i = Indicator(**data)
if not i.save():
raise AuthError('invalid token')
if raw:
return i
return i.to_dict()
def handle_indicators_create(self, token, data, id=None, client_id=None):
# this will raise AuthError if false
t = self.store.tokens.write(token)
if len(data) >= 1:
start_time = time.time()
if len(data) > 1:
logger.info('Upserting %d indicators..', len(data))
# this will raise AuthError if the groups don't match
r = self.store.indicators.upsert(t, data)
if len(data) > 1:
logger.info('Upserting %d indicators.. took %0.2f seconds',
len(data),
time.time() - start_time)
return r
if data['group'] not in t['groups']:
raise AuthError('unauthorized to write to group: %s' % g)
if not self.create_queue.get(token):
self.create_queue[token] = {'count': 0, "messages": []}
self.create_queue[token]['count'] += 1
self.create_queue_count += 1
self.create_queue[token]['last_activity'] = time.time()
if self.create_queue[token]['count'] > self.create_queue_limit:
self.create_queue[token]['messages'].append((id, client_id, data))
return MORE_DATA_NEEDED
def indicators_create(self, token, data):
if self.token_write(token):
if type(data) == dict:
data = [data]
self.logger.debug(data)
s = self.handle()
for d in data:
# namespace conflict with related self.tags
tags = d.get("tags", [])
if len(tags) > 0:
print type(tags)
if type(tags) == str or type(tags) == unicode:
if '.' in tags:
tags = tags.split(',')
else:
tags = [str(tags)]
del d['tags']
o = Indicator(**d)
s.add(o)
for t in tags:
t = Tag(tag=t, indicator=o)
s.add(t)
s.commit()
self.logger.debug('oid: {}'.format(o.id))
return o.id
else:
raise AuthError('invalid token')
def create(self, token, data, raw=False, bulk=False):
index = self._create_index()
expand_ip_idx(data)
id = i_to_id(data)
if data.get('group') and type(data['group']) != list:
data['group'] = [data['group']]
if bulk:
d = {'_index': index, '_type': 'indicator', '_source': data}
return d
data['meta'] = {}
data['meta']['index'] = index
data['meta']['id'] = id
i = Indicator(**data)
if not i.save():
raise AuthError('indicator exists')
if raw:
return i
return i.to_dict()
def _check_status(self, resp, expect=200):
if resp.status_code == 400:
r = json.loads(resp.text)
raise InvalidSearch(r['message'])
if resp.status_code == 401:
raise AuthError('unauthorized')
if resp.status_code == 404:
raise NotFound('not found')
if resp.status_code == 408:
raise TimeoutError('timeout')
if resp.status_code == 422:
msg = json.loads(resp.text)
raise SubmissionFailed(msg['message'])
if resp.status_code == 429:
raise CIFBusy('RateLimit exceeded')
if resp.status_code in [500, 501, 502, 503, 504]:
raise CIFBusy('system seems busy..')
if resp.status_code != expect:
msg = 'unknown: %s' % resp.content
raise RuntimeError(msg)
def indicators_create(self, token, data):
index = self._create_index()
self.logger.debug('index: {}'.format(index))
data['meta'] = {}
data['meta']['index'] = index
if resolve_itype(data['indicator']) == 'ipv4':
import re
match = re.search('^(\S+)\/(\d+)$', data['indicator'])
if match:
data['indicator_ipv4'] = match.group(1)
data['indicator_ipv4_mask'] = match.group(2)
else:
data['indicator_ipv4'] = data['indicator']
if type(data['group']) != list:
data['group'] = [data['group']]
self.logger.debug(data)
i = Indicator(**data)
self.logger.debug(i)
if i.save():
return i.__dict__['_d_']
else:
raise AuthError('invalid token')
def handle_indicators_search(self, token, data, **kwargs):
t = self.store.token_read(token)
if not t:
raise AuthError('invalid token')
if PYVERSION == 2:
if data.get('indicator'):
if isinstance(data['indicator'], str):
data['indicator'] = unicode(data['indicator'])
try:
x = self.store.indicators_search(data)
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
import traceback
logger.error(traceback.print_exc())
raise InvalidSearch('invalid search')
t = self.store.tokens_search({'token': token})
self._log_search(t, data)
if isinstance(x, GeneratorType):
x = list(x)
for xx in x:
if xx.get('message'):
xx['message'] = b64encode(xx['message']).encode('utf-8')
return x
def _check_indicator(self, i, t):
for e in REQUIRED_ATTRIBUTES:
if not i.get(e):
raise ValueError('missing %s' % e)
if i['group'] not in t['groups']:
raise AuthError('unable to write to %s' % i['group'])
return True
def check(self, token, k, v=True):
self._flush_cache()
if token in self._cache and self._cache[token].get(k):
return self._cache[token]
rv = list(self.search({'token': token, k: v}))
if len(rv) == 0:
raise AuthError('unauthorized')
self._cache[token] = rv[0]
return rv[0]
def handle_indicators_search(self, token, data):
if self.store.token_read(token):
self.logger.debug('searching')
try:
x = self.store.indicators_search(token, data)
except Exception as e:
self.logger.error(e)
raise InvalidSearch('invalid search')
else:
return x
else:
raise AuthError('invalid token')
def _check_status(self, resp, expect=200):
if resp.status_code == 401:
raise AuthError()
if resp.status_code == 404:
raise NotFound()
if resp.status_code == 408:
raise TimeoutError()
if resp.status_code == 422:
raise SubmissionFailed()
if resp.status_code != expect:
raise RuntimeError(resp.content)
def _get(self, uri, params={}):
if not uri.startswith('http'):
uri = self.remote + uri
body = self.session.get(uri, params=params, verify=self.verify_ssl)
if body.status_code > 303:
err = 'request failed: %s' % str(body.status_code)
logger.error(err)
if body.status_code == 401:
raise AuthError('invalid token')
elif body.status_code == 404:
err = 'not found'
raise RuntimeError(err)
elif body.status_code == 408:
raise TimeoutError('timeout')
else:
try:
err = json.loads(body.content).get('message')
raise RuntimeError(err)
except ValueError as e:
err = body.content
logger.error(err)
raise RuntimeError(err)
data = body.content
try:
data = zlib.decompress(b64decode(data))
except (TypeError, binascii.Error) as e:
pass
except Exception as e:
pass
msgs = json.loads(data.decode('utf-8'))
if not msgs.get('data'):
return msgs
if isinstance(msgs['data'], list):
for m in msgs['data']:
if m.get('message'):
try:
m['message'] = b64decode(m['message'])
except Exception as e:
pass
return msgs
def indicators_create(self, data, raw=False):
index = _create_index()
self.logger.debug('index: {}'.format(index))
data['meta'] = {}
data['meta']['index'] = index
itype = resolve_itype(data['indicator'])
if itype is 'ipv4':
match = re.search('^(\S+)\/(\d+)$', data['indicator'])
if match:
data['indicator_ipv4'] = match.group(1)
data['indicator_ipv4_mask'] = match.group(2)
else:
data['indicator_ipv4'] = data['indicator']
elif itype is 'ipv6':
match = re.search('^(\S+)\/(\d+)$', data['indicator'])
if match:
data['indicator_ipv6'] = binascii.b2a_hex(
socket.inet_pton(socket.AF_INET6,
match.group(1))).decode('utf-8')
data['indicator_ipv6_mask'] = match.group(2)
else:
data['indicator_ipv6'] = binascii.b2a_hex(
socket.inet_pton(socket.AF_INET6,
data['indicator'])).decode('utf-8')
if type(data['group']) != list:
data['group'] = [data['group']]
if data.get('message'):
try:
data['message'] = str(b64decode(data['message']))
except (TypeError, binascii.Error) as e:
pass
i = Indicator(**data)
if i.save():
if raw:
return i
else:
return i.__dict__['_d_']
else:
raise AuthError('invalid token')
def _recv(self, decode=True, close=True):
mtype, data = Msg().recv(self.socket)
if close:
self.socket.close()
if not decode:
return data
data = json.loads(data)
if data.get('message') == 'unauthorized':
raise AuthError()
if data.get('message') == 'busy':
raise CIFBusy()
if data.get('message') == 'invalid search':
raise InvalidSearch()
if data.get('status') != 'success':
raise RuntimeError(data.get('message'))
if data.get('data') is None:
raise RuntimeError('invalid response')
if isinstance(data.get('data'), bool):
return data['data']
# is this a straight up elasticsearch string?
if data['data'] == '{}':
return []
if isinstance(data['data'], basestring) and data['data'].startswith('{"hits":{"hits":[{"_source":'):
data['data'] = json.loads(data['data'])
data['data'] = [r['_source'] for r in data['data']['hits']['hits']]
try:
data['data'] = zlib.decompress(data['data'])
except (zlib.error, TypeError):
pass
return data.get('data')
def _check_indicator(i, t):
if not i.get('group'):
i['group'] = t['groups'][0]
if not i.get('provider'):
i['provider'] = t['username']
if not i.get('tags'):
i['tags'] = 'suspicious'
if not i.get('reported_at'):
i['reported_at'] = arrow.utcnow().datetime
for e in REQUIRED_ATTRIBUTES:
if not i.get(e):
raise ValueError('missing %s' % e)
if i['group'] not in t['groups']:
raise AuthError('unable to write to %s' % i['group'])
return True
def check_token_perms(self, mtype, token, data):
"""Check token perms/groups and raise AuthError if issues
:param mtype(str)
:param token(list) - from self.auth.handle_token_search
:param data(str) - a str dumped dict from Msg()
:return token(dict) - matched token dict
"""
if not token:
raise AuthError('Auth: invalid token provided to API')
# if more than one token comes back, shenanigans
elif len(token) > 1:
raise AuthError('multiple token matches during auth. possible wildcard attempt?')
else:
token = token[0]
# check action (mtype) against token perms
if mtype.startswith('tokens') or mtype.endswith('delete'):
if not token.get('admin'):
raise AuthError('Auth: admin function attempted but supplied token had no admin permission')
elif mtype.endswith('create') or mtype.endswith('write'):
if not token.get('write'):
raise AuthError('Auth: write function attempted but supplied token had no write permission')
elif mtype == 'indicators_search':
if not token.get('read'):
raise AuthError('Auth: read function attempt but supplied token had no read permission')
# check action (mtype) against token groups
if mtype == 'indicators_create':
data = json.loads(data)
if isinstance(data, dict):
data = [data]
for i in data:
if i.get('group', 'everyone') not in token['groups']:
raise AuthError('Auth: indicator function attempt {} but supplied indicator group {} did not match user token groups {}'.format(mtype, i.get('group', 'everyone'), token['groups']))
# for indicators_search and _delete, they use plural "groups"
return token
def handle_indicators_create(self, token, data):
if self.store.token_write(token):
return self.store.indicators_create(token, data)
else:
raise AuthError('invalid token')
def handle_indicators_search(self, token, data):
if self.store.token_read(token):
self.logger.debug('searching')
return self.store.indicators_search(token, data)
else:
raise AuthError('invalid token')
def handle_tokens_edit(self, token, data, **kwargs):
if self.store.tokens.admin(token):
return self.store.tokens.edit(data)
raise AuthError('invalid token')
def handle_tokens_create(self, token, data, **kwargs):
if self.store.tokens.admin(token):
logger.debug('tokens_create')
return self.store.tokens.create(data)
raise AuthError('invalid token')
def handle_indicators_create(self,
token,
data,
id=None,
client_id=None,
flush=False):
# this will raise AuthError if false
t = self.store.tokens.write(token)
if len(data) > 1 or t['username'] == 'admin':
start_time = time.time()
logger.info('attempting to insert %d indicators..', len(data))
# this will raise AuthError if the groups don't match
if isinstance(data, dict):
data = [data]
for i in data:
if not i.get('group'):
i['group'] = 'everyone'
# optionally force provider to match username with exceptions
if i.get('provider') and STRICT_PROVIDERS and not t[
'username'] in STRICT_PROVIDERS_EXCEPTIONS:
i['provider'] = t['username']
if not i.get('provider') or i['provider'] == '':
i['provider'] = t['username']
if i['group'] not in t['groups']:
raise AuthError('unable to write to %s' % i['group'])
if not i.get('tags'):
i['tags'] = ['suspicious']
if i.get('message'):
try:
i['message'] = str(b64decode(data['message']))
except (TypeError, binascii.Error) as e:
pass
self._timestamps_fix(i)
n = self.store.indicators.upsert(t, data, flush=flush)
t = time.time() - start_time
logger.info(
'actually inserted %d indicators.. took %0.2f seconds (%0.2f/sec)',
n, t, (n / t))
return n
data = data[0]
if data['group'] not in t['groups']:
raise AuthError('unauthorized to write to group: %s' %
data['group'])
if data.get('message'):
try:
data['message'] = str(b64decode(data['message']))
except (TypeError, binascii.Error) as e:
pass
if not self.create_queue.get(token):
self.create_queue[token] = {'count': 0, "messages": []}
self.create_queue[token]['count'] += 1
self.create_queue_count += 1
self.create_queue[token]['last_activity'] = time.time()
self.create_queue[token]['messages'].append((id, client_id, [data]))
return MORE_DATA_NEEDED
def _flush_create_queue(self):
for t in self.create_queue:
if len(self.create_queue[t]['messages']) == 0:
continue
logger.debug('flushing queue...')
data = [msg[0] for _, _, msg in self.create_queue[t]['messages']]
_t = self.store.tokens.write(t)
try:
start_time = time.time()
logger.info('attempting to insert %d indicators..', len(data))
# this will raise AuthError if the groups don't match
if isinstance(data, dict):
data = [data]
for i in data:
if not i.get('group'):
i['group'] = 'everyone'
# optionally force provider to match username with exceptions
if i.get('provider') and STRICT_PROVIDERS and not _t[
'username'] in STRICT_PROVIDERS_EXCEPTIONS:
i['provider'] = _t['username']
if not i.get('provider') or i['provider'] == '':
i['provider'] = _t['username']
if i['group'] not in _t['groups']:
raise AuthError('unable to write to %s' % i['group'])
if not i.get('tags'):
i['tags'] = ['suspicious']
if i.get('message'):
try:
i['message'] = str(b64decode(data['message']))
except (TypeError, binascii.Error) as e:
pass
self._timestamps_fix(i)
n = self.store.indicators.upsert(_t, data)
t_time = time.time() - start_time
logger.info(
'actually inserted %d indicators.. took %0.2f seconds (%0.2f/sec)',
n, t_time, (n / t_time))
if n == 0:
rv = {'status': 'failed', 'message': 'invalid indicator'}
else:
rv = {"status": "success", "data": n}
except AuthError as e:
rv = {'status': 'failed', 'message': 'unauthorized'}
except StoreLockError:
rv = {'status': 'failed', 'message': 'busy'}
for id, client_id, _ in self.create_queue[t]['messages']:
Msg(id=id,
client_id=client_id,
mtype=Msg.INDICATORS_CREATE,
data=rv)
if rv['status'] == 'success':
self.store.tokens.update_last_activity_at(
t,
arrow.utcnow().datetime)
logger.debug('queue flushed..')
def handle_tokens_delete(self, token, data, **kwargs):
if self.store.token_admin(token):
return self.store.tokens_delete(data)
else:
raise AuthError('invalid token')
def handle_tokens_create(self, token, data):
if self.store.token_admin(token):
self.logger.debug('tokens_create')
return self.store.tokens_create(data)
else:
raise AuthError('invalid token')
def handle_tokens_edit(self, token, data):
if self.store.token_admin(token):
return self.store.token_edit(data)
else:
raise AuthError('invalid token')
def handle_indicators_search(self, token, data, **kwargs):
if PYVERSION == 2:
if data.get('indicator'):
if isinstance(data['indicator'], str):
data['indicator'] = unicode(data['indicator'])
# token acl check
if token.get('acl') and token.get('acl') != ['']:
if data.get('itype') and data.get('itype') not in token['acl']:
raise AuthError('unauthorized to access itype {}'.format(
data['itype']))
if not data.get('itype'):
data['itype'] = token['acl']
# verify group filter matches token permissions
if data.get('groups') and (not token.get('admin')
or token.get('admin') == ''):
if isinstance(data['groups'], basestring):
q_groups = [g.strip() for g in data['groups'].split(',')]
elif isinstance(data['groups'], list):
q_groups = data['groups']
gg = []
for g in q_groups:
if AUTH_ENABLED:
if g in token['groups']:
gg.append(g)
else:
gg.append(g)
if gg:
data['groups'] = gg
else:
data['groups'] = '{}'
if not data.get('reporttime'):
if data.get('days'):
now = arrow.utcnow()
data['reporttimeend'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.shift(days=-int(data['days']))
data['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
if data.get('hours'):
now = arrow.utcnow()
data['reporttimeend'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.shift(hours=-int(data['hours']))
data['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
s = time.time()
self._log_search(token, data)
try:
x = self.store.indicators.search(token, data)
logger.debug('done')
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
logger.error(traceback.print_exc())
raise InvalidSearch(': {}'.format(e))
logger.debug('%s' % (time.time() - s))
# for xx in x:
# if xx.get('message'):
# xx['message'] = b64encode(xx['message']).encode('utf-8')
return x