def handle_indicators_search(self, token, data, **kwargs):
t = self.store.tokens.read(token)
try:
self._log_search(t, data)
except TypeError:
raise InvalidSearch('invalid search')
except Exception as e:
logger.error(e)
try:
x = self.store.indicators.search(t, data)
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
import traceback
traceback.print_exc()
raise InvalidSearch('invalid search')
if isinstance(x, GeneratorType):
x = list(x)
return x
def _filter_indicator(self, filters, s):
for k, v in list(filters.items()):
if k not in VALID_FILTERS:
del filters[k]
if not filters.get('indicator'):
return s
i = filters.pop('indicator')
if PYVERSION == 2:
if isinstance(i, str):
i = unicode(i)
try:
itype = resolve_itype(i)
except InvalidIndicator as e:
logger.error(e)
s = s.join(Message).filter(Indicator.Message.like(
'%{}%'.format(i)))
return s
if itype in ['fqdn', 'email', 'url']:
s = s.filter(Indicator.indicator == i)
return s
if itype == 'ipv4':
ip = ipaddress.IPv4Network(i)
mask = ip.prefixlen
if mask < 8:
raise InvalidSearch('prefix needs to be >= 8')
start = str(ip.network_address)
end = str(ip.broadcast_address)
logger.debug('{} - {}'.format(start, end))
s = s.join(Ipv4).filter(Ipv4.ipv4 >= start)
s = s.filter(Ipv4.ipv4 <= end)
return s
if itype == 'ipv6':
ip = ipaddress.IPv6Network(i)
mask = ip.prefixlen
if mask < 32:
raise InvalidSearch('prefix needs to be >= 32')
start = str(ip.network_address)
end = str(ip.broadcast_address)
logger.debug('{} - {}'.format(start, end))
s = s.join(Ipv6).filter(Ipv6.ip >= start)
s = s.filter(Ipv6.ip <= end)
return s
raise InvalidIndicator
def handle_indicators_search(self, token, data, **kwargs):
t = self.store.tokens.read(token)
if data.get('indicator'):
# python2
try:
if isinstance(data['indicator'], str):
data['indicator'] = unicode(data['indicator'])
except:
pass
self._log_search(t, data)
try:
x = self.store.indicators.search(t, data)
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
import traceback
traceback.print_exc()
raise InvalidSearch('invalid search')
return x
def _recv(self, decode=True):
mtype, data = Msg().recv(self.socket)
if not decode:
return data
data = json.loads(data)
if data.get('message') == 'unauthorized':
raise AuthError()
if data.get('message') == 'busy':
raise CIFBusy()
if data.get('message') == 'invalid search':
raise InvalidSearch()
if data.get('status') != 'success':
raise RuntimeError(data.get('message'))
try:
data['data'] = zlib.decompress(data['data'])
except (zlib.error, TypeError):
pass
return data.get('data')
def _get(self, uri, params={}):
if not uri.startswith('http'):
uri = self.remote + uri
resp = self.session.get(uri,
params=params,
verify=self.verify_ssl,
timeout=self.timeout)
self._check_status(resp, expect=200)
data = resp.content
s = (int(resp.headers['Content-Length']) / 1024 / 1024)
logger.info('processing %.2f megs' % s)
msgs = json.loads(data.decode('utf-8'))
if not msgs.get('status') and not msgs.get('message') == 'success':
raise RuntimeError(msgs)
if msgs.get('status') and msgs['status'] == 'failure':
raise InvalidSearch(msgs['message'])
if isinstance(msgs.get('data'), list):
for m in msgs['data']:
if m.get('message'):
try:
m['message'] = b64decode(m['message'])
except Exception as e:
pass
return msgs
def _check_data(self, msgs):
if isinstance(msgs, list):
return msgs
if msgs.get('status', False) not in ['success', 'failure']:
raise RuntimeError(msgs)
if msgs.get('status') == 'failure':
raise InvalidSearch(msgs['message'])
if msgs['data'] == '{}':
msgs['data'] = []
# check to see if it's straight elasticsearch json
if isinstance(msgs['data'], basestring) and msgs['data'].startswith(
'{"hits":{"hits":[{"_source":'):
msgs['data'] = json.loads(msgs['data'])
msgs['data'] = [r['_source'] for r in msgs['data']['hits']['hits']]
if not isinstance(msgs['data'], list):
msgs['data'] = [msgs['data']]
for m in msgs['data']:
if not isinstance(m, dict):
continue
if not m.get('message'):
continue
try:
m['message'] = b64decode(m['message'])
except Exception as e:
pass
return msgs
def _check_status(self, resp, expect=200):
if resp.status_code == 400:
r = json.loads(resp.text)
raise InvalidSearch(r['message'])
if resp.status_code == 401:
raise AuthError('unauthorized')
if resp.status_code == 404:
raise NotFound('not found')
if resp.status_code == 408:
raise TimeoutError('timeout')
if resp.status_code == 422:
msg = json.loads(resp.text)
raise SubmissionFailed(msg['message'])
if resp.status_code == 429:
raise CIFBusy('RateLimit exceeded')
if resp.status_code in [500, 501, 502, 503, 504]:
raise CIFBusy('system seems busy..')
if resp.status_code != expect:
msg = 'unknown: %s' % resp.content
raise RuntimeError(msg)
def handle_indicators_search(self, token, data, **kwargs):
t = self.store.token_read(token)
if not t:
raise AuthError('invalid token')
if PYVERSION == 2:
if data.get('indicator'):
if isinstance(data['indicator'], str):
data['indicator'] = unicode(data['indicator'])
try:
x = self.store.indicators_search(data)
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
import traceback
logger.error(traceback.print_exc())
raise InvalidSearch('invalid search')
t = self.store.tokens_search({'token': token})
self._log_search(t, data)
if isinstance(x, GeneratorType):
x = list(x)
for xx in x:
if xx.get('message'):
xx['message'] = b64encode(xx['message']).encode('utf-8')
return x
def _filter_terms(self, filters, s):
# TODO also you should do for k, v in filters.items():
# iteritems()?
for k in filters:
if k in ['nolog', 'days', 'hours', 'groups', 'limit']:
continue
if k == 'reporttime':
if ',' in filters[k]:
start, end = filters[k].split(',')
s = s.filter(
Indicator.reporttime >= arrow.get(start).datetime)
s = s.filter(
Indicator.reporttime <= arrow.get(end).datettime)
else:
s = s.filter(
Indicator.reporttime >= arrow.get(filters[k]).datetime)
elif k == 'reporttimeend':
s = s.filter(Indicator.reporttime <= filters[k])
elif k == 'tags':
s = s.outerjoin(Tag).filter(Tag.tag == filters[k])
elif k == 'confidence':
if ',' in str(filters[k]):
start, end = str(filters[k]).split(',')
s = s.filter(Indicator.confidence >= float(start))
s = s.filter(Indicator.confidence <= float(end))
else:
s = s.filter(Indicator.confidence >= float(filters[k]))
elif k == 'itype':
s = s.filter(Indicator.itype == filters[k])
elif k == 'provider':
s = s.filter(Indicator.provider == filters[k])
elif k == 'asn':
s = s.filter(Indicator.asn == filters[k])
elif k == 'asn_desc':
s = s.filter(Indicator.asn_desc.like('%{}%'.format(
filters[k])))
elif k == 'cc':
s = s.filter(Indicator.cc == filters[k])
elif k == 'rdata':
s = s.filter(Indicator.rdata == filters[k])
elif k == 'region':
s = s.filter(Indicator.region == filters[k])
else:
raise InvalidSearch('invalid filter: %s' % k)
return s
def indicators_search(self, token, filters):
# build filters with elasticsearch-dsl
# http://elasticsearch-dsl.readthedocs.org/en/latest/search_dsl.html
limit = filters.get('limit')
if limit:
del filters['limit']
else:
limit = LIMIT
nolog = filters.get('nolog')
if nolog:
del filters['nolog']
timeout = TIMEOUT
s = Indicator.search()
s = s.params(size=limit, timeout=timeout)
#s = s.sort('-reporttime')
q_filters = {}
for f in VALID_FILTERS:
if filters.get(f):
q_filters[f] = filters[f]
if q_filters.get('indicator'):
itype = resolve_itype(q_filters['indicator'])
if itype == 'ipv4':
ip = ipaddress.IPv4Network(q_filters['indicator'])
mask = ip.prefixlen
if mask < 8:
raise InvalidSearch(
'prefix needs to be greater than or equal to 8')
start = str(ip.network_address)
end = str(ip.broadcast_address)
s = s.filter('range',
indicator_ipv4={
'gte': start,
'lte': end
})
del q_filters['indicator']
for f in q_filters:
kwargs = {f: q_filters[f]}
s = s.filter('term', **kwargs)
try:
rv = s.execute()
except elasticsearch.exceptions.RequestError as e:
self.logger.error(e)
return []
try:
return [x['_source'] for x in rv.hits.hits]
except KeyError:
return []
def _filter_indicator(self, q_filters, s):
if not q_filters.get('indicator'):
return s
i = q_filters.pop('indicator')
try:
itype = resolve_itype(i)
except InvalidIndicator:
s = s.query("match", message=i)
return s
if itype in ('email', 'url', 'fqdn'):
s = s.filter('term', indicator=i)
return s
if itype is 'ipv4':
ip = ipaddress.IPv4Network(i)
mask = ip.prefixlen
if mask < 8:
raise InvalidSearch(
'prefix needs to be greater than or equal to 8')
start = str(ip.network_address)
end = str(ip.broadcast_address)
s = s.filter('range', indicator_ipv4={'gte': start, 'lte': end})
return s
if itype is 'ipv6':
ip = ipaddress.IPv6Network(i)
mask = ip.prefixlen
if mask < 32:
raise InvalidSearch(
'prefix needs to be greater than or equal to 32')
start = binascii.b2a_hex(
socket.inet_pton(socket.AF_INET6,
str(ip.network_address))).decode('utf-8')
end = binascii.b2a_hex(
socket.inet_pton(socket.AF_INET6,
str(ip.broadcast_address))).decode('utf-8')
s = s.filter('range', indicator_ipv6={'gte': start, 'lte': end})
return s
def _filter_ipv4(s, i):
ip = ipaddress.IPv4Network(i)
mask = ip.prefixlen
if mask < 8:
raise InvalidSearch('prefix needs to be greater than or equal to 8')
start = str(ip.network_address)
end = str(ip.broadcast_address)
s = s.filter('range', indicator_ipv4={'gte': start, 'lte': end})
return s
def handle_indicators_search(self, token, data):
if self.store.token_read(token):
self.logger.debug('searching')
try:
x = self.store.indicators_search(token, data)
except Exception as e:
self.logger.error(e)
raise InvalidSearch('invalid search')
else:
return x
else:
raise AuthError('invalid token')
def handle_stats_search(self, token, data, **kwargs):
t = self.store.tokens.read(token)
try:
x = self.store.indicators.stats_search(t, data)
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
traceback.print_exc()
raise InvalidSearch('invalid search')
return x
def _filter_ipv6(s, i):
ip = ipaddress.IPv6Network(i)
mask = ip.prefixlen
if mask < 32:
raise InvalidSearch('prefix needs to be greater than or equal to 32')
start = binascii.b2a_hex(
socket.inet_pton(socket.AF_INET6,
str(ip.network_address))).decode('utf-8')
end = binascii.b2a_hex(
socket.inet_pton(socket.AF_INET6,
str(ip.broadcast_address))).decode('utf-8')
s = s.filter('range', indicator_ipv6={'gte': start, 'lte': end})
return s
def handle_indicators_search(self, token, data, **kwargs):
t = self.store.tokens.read(token)
if PYVERSION == 2:
if data.get('indicator'):
if isinstance(data['indicator'], str):
data['indicator'] = unicode(data['indicator'])
if not data.get('reporttime'):
if data.get('days'):
now = arrow.utcnow()
data['reporttimeend'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.replace(days=-int(data['days']))
data['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
if data.get('hours'):
now = arrow.utcnow()
data['reporttimeend'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.replace(hours=-int(data['hours']))
data['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
s = time.time()
self._log_search(t, data)
try:
x = self.store.indicators.search(t, data)
logger.debug('done')
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
logger.error(traceback.print_exc())
raise InvalidSearch('invalid search')
logger.debug('%s' % (time.time() - s))
# for xx in x:
# if xx.get('message'):
# xx['message'] = b64encode(xx['message']).encode('utf-8')
return x
def _recv(self, decode=True, close=True):
mtype, data = Msg().recv(self.socket)
if close:
self.socket.close()
if not decode:
return data
data = json.loads(data)
if data.get('message') == 'unauthorized':
raise AuthError()
if data.get('message') == 'busy':
raise CIFBusy()
if data.get('message') == 'invalid search':
raise InvalidSearch()
if data.get('status') != 'success':
raise RuntimeError(data.get('message'))
if data.get('data') is None:
raise RuntimeError('invalid response')
if isinstance(data.get('data'), bool):
return data['data']
# is this a straight up elasticsearch string?
if data['data'] == '{}':
return []
if isinstance(data['data'], basestring) and data['data'].startswith('{"hits":{"hits":[{"_source":'):
data['data'] = json.loads(data['data'])
data['data'] = [r['_source'] for r in data['data']['hits']['hits']]
try:
data['data'] = zlib.decompress(data['data'])
except (zlib.error, TypeError):
pass
return data.get('data')
def _filter_terms(self, filters, s):
# TODO also you should do for k, v in filters.items():
# iteritems()?
for k in filters:
if k in ['nolog', 'days', 'hours', 'groups', 'limit']:
continue
if k == 'reporttime':
if ',' in filters[k]:
start, end = filters[k].split(',')
s = s.filter(Indicator.reporttime >= start)
s = s.filter(Indicator.reporttime <= end)
else:
s = s.filter(Indicator.reporttime >= filters[k])
elif k == 'reporttimeend':
s = s.filter(Indicator.reporttime <= filters[k])
elif k == 'tags':
s = s.join(Tag).filter(Tag.tag == filters[k])
elif k == 'confidence':
if ',' in str(filters[k]):
start, end = str(filters[k]).split(',')
s = s.filter(Indicator.confidence >= float(start))
s = s.filter(Indicator.confidence <= float(end))
else:
s = s.filter(Indicator.confidence >= filters[k])
elif k == 'itype':
s = s.filter(Indicator.itype == filters[k])
elif k == 'provider':
s = s.filter(Indicator.provider == filters[k])
else:
raise InvalidSearch('invalid filter: %s' % k)
return s
def filter_reporttime(s, filter):
if not filter.get('reporttime'):
return s
high = 'now/m'
# if passed 'days' or 'hours', preferentially use that for ES filtering/caching
if filter.get('days') or filter.get('hours'):
if filter.get('hours'):
lookback_amount = filter.pop('hours')
lookback_unit = 'h'
elif filter.get('days'):
lookback_amount = filter.pop('days')
lookback_unit = 'd'
try:
lookback_amount = int(lookback_amount)
except Exception as e:
raise InvalidSearch(
'Lookback time filter {}{} is not a valid time'.format(
lookback_amount, lookback_unit))
# don't put spaces in relative date math operator query to make it easier to read. ES hates that and will error.
low = 'now/m-{}{}'.format(lookback_amount, lookback_unit)
# no relative 'days' or 'hours' params, so fallback to 'reporttime'
else:
c = filter.pop('reporttime')
if PYVERSION == 2:
if type(c) == unicode:
c = str(c)
if isinstance(c, basestring) and ',' in c:
low, high = c.split(',')
else:
low = c
low = arrow.get(low).datetime
s = s.filter('range', reporttime={'gte': low, 'lte': high})
return s
def filter_build(s, filters, token=None):
limit = filters.get('limit')
if limit and int(limit) > WINDOW_LIMIT:
raise InvalidSearch(
'Request limit should be <= server threshold of {} but was set to {}'
.format(WINDOW_LIMIT, limit))
q_filters = {}
for f in VALID_FILTERS:
if filters.get(f):
q_filters[f] = filters[f]
s = filter_provider(s, q_filters)
s = filter_confidence(s, q_filters)
s = filter_id(s, q_filters)
s = filter_rdata(s, q_filters)
# treat indicator as special, transform into Search
s = filter_indicator(s, q_filters)
s = filter_reporttime(s, q_filters)
# transform all other filters into term=
s = filter_terms(s, q_filters)
if q_filters.get('groups'):
s = filter_groups(s, q_filters)
else:
if token and (not token.get('admin') or token.get('admin') == ''):
s = filter_groups(s, {}, token=token)
if q_filters.get('tags'):
s = filter_tags(s, q_filters)
return s
def handle_indicators_search(self, token, data, **kwargs):
if PYVERSION == 2:
if data.get('indicator'):
if isinstance(data['indicator'], str):
data['indicator'] = unicode(data['indicator'])
# token acl check
if token.get('acl') and token.get('acl') != ['']:
if data.get('itype') and data.get('itype') not in token['acl']:
raise AuthError('unauthorized to access itype {}'.format(
data['itype']))
if not data.get('itype'):
data['itype'] = token['acl']
# verify group filter matches token permissions
if data.get('groups') and (not token.get('admin')
or token.get('admin') == ''):
if isinstance(data['groups'], basestring):
q_groups = [g.strip() for g in data['groups'].split(',')]
elif isinstance(data['groups'], list):
q_groups = data['groups']
gg = []
for g in q_groups:
if AUTH_ENABLED:
if g in token['groups']:
gg.append(g)
else:
gg.append(g)
if gg:
data['groups'] = gg
else:
data['groups'] = '{}'
if not data.get('reporttime'):
if data.get('days'):
now = arrow.utcnow()
data['reporttimeend'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.shift(days=-int(data['days']))
data['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
if data.get('hours'):
now = arrow.utcnow()
data['reporttimeend'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.shift(hours=-int(data['hours']))
data['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
s = time.time()
self._log_search(token, data)
try:
x = self.store.indicators.search(token, data)
logger.debug('done')
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
logger.error(traceback.print_exc())
raise InvalidSearch(': {}'.format(e))
logger.debug('%s' % (time.time() - s))
# for xx in x:
# if xx.get('message'):
# xx['message'] = b64encode(xx['message']).encode('utf-8')
return x
def indicators_search(self, filters, limit=500):
self.logger.debug('running search')
limit = filters.pop('limit', limit)
nolog = filters.pop('nolog', False)
q_filters = {}
for f in VALID_FILTERS:
if filters.get(f):
q_filters[f] = filters[f]
s = self.handle().query(Indicator)
filters = q_filters
sql = []
if filters.get('indicator'):
try:
itype = resolve_itype(filters['indicator'])
self.logger.debug('itype %s' % itype)
if itype == 'ipv4':
if PYVERSION < 3 and (filters['indicator'], str):
filters['indicator'] = filters['indicator'].decode(
'utf-8')
ip = ipaddress.IPv4Network(filters['indicator'])
mask = ip.prefixlen
if mask < 8:
raise InvalidSearch('prefix needs to be >= 8')
start = str(ip.network_address)
end = str(ip.broadcast_address)
self.logger.debug('{} - {}'.format(start, end))
s = s.join(Ipv4).filter(Ipv4.ipv4 >= start)
s = s.filter(Ipv4.ipv4 <= end)
elif itype == 'ipv6':
if PYVERSION < 3 and (filters['indicator'], str):
filters['indicator'] = filters['indicator'].decode(
'utf-8')
ip = ipaddress.IPv6Network(filters['indicator'])
mask = ip.prefixlen
if mask < 32:
raise InvalidSearch('prefix needs to be >= 32')
start = str(ip.network_address)
end = str(ip.broadcast_address)
self.logger.debug('{} - {}'.format(start, end))
s = s.join(Ipv6).filter(Ipv6.ip >= start)
s = s.filter(Ipv6.ip <= end)
elif itype in ('fqdn', 'email', 'url'):
sql.append("indicator = '{}'".format(filters['indicator']))
except InvalidIndicator as e:
self.logger.error(e)
sql.append("message LIKE '%{}%'".format(filters['indicator']))
s = s.join(Message)
del filters['indicator']
for k in filters:
if k == 'reporttime':
sql.append("{} >= '{}'".format('reporttime', filters[k]))
elif k == 'reporttimeend':
sql.append("{} <= '{}'".format('reporttime', filters[k]))
elif k == 'tags':
sql.append("tags.tag == '{}'".format(filters[k]))
elif k == 'confidence':
sql.append("{} >= '{}'".format(k, filters[k]))
else:
sql.append("{} = '{}'".format(k, filters[k]))
sql = ' AND '.join(sql)
if sql:
self.logger.debug(sql)
if filters.get('tags'):
s = s.join(Tag)
rv = s.order_by(desc(Indicator.reporttime)).filter(
text(sql)).limit(limit)
return [self._as_dict(x) for x in rv]
def _get(self, uri, params={}, retry=True):
if not uri.startswith('http'):
uri = self.remote + uri
resp = self.session.get(uri,
params=params,
verify=self.verify_ssl,
timeout=self.timeout)
n = RETRIES
try:
self._check_status(resp, expect=200)
n = 0
except Exception as e:
if resp.status_code == 429 or resp.status_code in [
500, 501, 502, 503, 504
]:
logger.error(e)
else:
raise e
while n != 0:
logger.warning(
'setting random retry interval to spread out the load')
logger.warning('retrying in %.00fs' % RETRIES_DELAY)
sleep(RETRIES_DELAY)
resp = self.session.get(uri,
params=params,
verify=self.verify_ssl,
timeout=self.timeout)
if resp.status_code == 200:
break
if n == 0:
raise CIFBusy('system seems busy.. try again later')
data = resp.content
s = (int(resp.headers['Content-Length']) / 1024 / 1024)
logger.info('processing %.2f megs' % s)
msgs = json.loads(data.decode('utf-8'))
if msgs.get('data') and msgs['data'] == '{}':
msgs['data'] = []
if msgs.get('data') and isinstance(
msgs['data'], basestring) and msgs['data'].startswith(
'{"hits":{"hits":[{"_source":'):
msgs['data'] = json.loads(msgs['data'])
msgs['data'] = [r['_source'] for r in msgs['data']['hits']['hits']]
if not msgs.get('status') and not msgs.get('message') == 'success':
raise RuntimeError(msgs)
if msgs.get('status') and msgs['status'] == 'failed':
raise InvalidSearch(msgs['message'])
if isinstance(msgs.get('data'), list):
for m in msgs['data']:
if m.get('message'):
try:
m['message'] = b64decode(m['message'])
except Exception as e:
pass
return msgs
def _filter_indicator(self, filters, s):
for k, v in list(filters.items()):
if k not in VALID_FILTERS:
del filters[k]
if not filters.get('indicator'):
return s
i = filters.pop('indicator')
try:
itype = resolve_itype(i)
except TypeError as e:
logger.error(e)
s = s.join(Message).filter(Indicator.Message.like('%{}%'.format(i)))
return s
if itype == 'email':
s = s.join(Email).filter(or_(
Email.email.like('%.{}'.format(i)),
Email.email == i)
)
return s
if itype == 'ipv4':
ip = ipaddress.IPv4Network(i)
mask = ip.prefixlen
if mask < 8:
raise InvalidSearch('prefix needs to be >= 8')
start = str(ip.network_address)
end = str(ip.broadcast_address)
logger.debug('{} - {}'.format(start, end))
s = s.join(Ipv4).filter(Ipv4.ipv4 >= start)
s = s.filter(Ipv4.ipv4 <= end)
return s
if itype == 'ipv6':
ip = ipaddress.IPv6Network(i)
mask = ip.prefixlen
if mask < 32:
raise InvalidSearch('prefix needs to be >= 32')
start = str(ip.network_address)
end = str(ip.broadcast_address)
logger.debug('{} - {}'.format(start, end))
s = s.join(Ipv6).filter(Ipv6.ip >= start)
s = s.filter(Ipv6.ip <= end)
return s
if itype == 'fqdn':
s = s.join(Fqdn).filter(or_(
Fqdn.fqdn.like('%.{}'.format(i)),
Fqdn.fqdn == i)
)
return s
if itype == 'url':
s = s.join(Url).filter(Url.url == i)
return s
if itype in HASH_TYPES:
s = s.join(Hash).filter(Hash.hash == str(i))
return s
raise ValueError
def _filter_terms(self, filters, s):
for k, v in filters.items():
if k in ['nolog', 'days', 'hours', 'groups', 'limit', 'feed']:
continue
if k == 'reported_at':
if ',' in v:
start, end = v.split(',')
s = s.filter(
Indicator.reported_at >= arrow.get(start).datetime)
s = s.filter(
Indicator.reported_at <= arrow.get(end).datetime)
else:
s = s.filter(
Indicator.reported_at >= arrow.get(v).datetime)
elif k == 'tags':
t = v.split(',')
s = s.outerjoin(Tag)
s = s.filter(or_(Tag.tag == tt for tt in t))
elif k == 'confidence':
if ',' in str(v):
start, end = str(v).split(',')
s = s.filter(Indicator.confidence >= float(start))
s = s.filter(Indicator.confidence <= float(end))
else:
s = s.filter(Indicator.confidence >= float(v))
elif k == 'probability':
if ',' in str(v):
start, end = str(v).split(',')
if start == 0:
s = s.filter(
or_(Indicator.probability >= float(start),
Indicator.probability == None))
s = s.filter(Indicator.probability <= float(end))
else:
s = s.filter(Indicator.probability >= float(start))
s = s.filter(Indicator.probability <= float(end))
else:
if float(v) == 0:
s = s.filter(
or_(Indicator.probability == None,
Indicator.probability >= float(v)))
else:
s = s.filter(Indicator.probability >= float(v))
elif k == 'itype':
s = s.filter(Indicator.itype == v)
elif k == 'provider':
s = s.filter(Indicator.provider == v)
elif k == 'asn':
s = s.filter(Indicator.asn == v)
elif k == 'asn_desc':
s = s.filter(Indicator.asn_desc.like('%{}%'.format(v)))
elif k == 'cc':
s = s.filter(Indicator.cc == v)
elif k == 'rdata':
s = s.filter(Indicator.rdata == v)
elif k == 'region':
s = s.filter(Indicator.region == v)
elif k == 'related':
s = s.filter(Indicator.related == v)
elif k == 'uuid':
s = s.filter(Indicator.uuid == v)
else:
raise InvalidSearch('invalid filter: %s' % k)
return s
def handle_indicators_search(self, token, data, **kwargs):
t = self.store.tokens.read(token)
if PYVERSION == 2:
if data.get('indicator'):
if isinstance(data['indicator'], str):
data['indicator'] = unicode(data['indicator'])
# verify group filter matches token permissions
if data.get('groups'):
q_groups = [g.strip() for g in data['groups'].split(',')]
gg = []
for g in q_groups:
if g in t['groups']:
gg.append(g)
if gg:
data['groups'] = gg
else:
data['groups'] = '{}'
if not data.get('reporttime'):
if data.get('days'):
now = arrow.utcnow()
data['reporttimeend'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.shift(days=-int(data['days']))
data['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
if data.get('hours'):
now = arrow.utcnow()
data['reporttimeend'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.shift(hours=-int(data['hours']))
data['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
s = time.time()
self._log_search(t, data)
try:
x = self.store.indicators.search(t, data)
logger.debug('done')
except Exception as e:
logger.error(e)
if logger.getEffectiveLevel() == logging.DEBUG:
logger.error(traceback.print_exc())
raise InvalidSearch(': {}'.format(e))
logger.debug('%s' % (time.time() - s))
# for xx in x:
# if xx.get('message'):
# xx['message'] = b64encode(xx['message']).encode('utf-8')
return x
def indicators_search(self, filters, sort=None, raw=False):
# build filters with elasticsearch-dsl
# http://elasticsearch-dsl.readthedocs.org/en/latest/search_dsl.html
limit = filters.get('limit')
if limit:
del filters['limit']
else:
limit = LIMIT
nolog = filters.get('nolog')
if nolog:
del filters['nolog']
timeout = TIMEOUT
s = Indicator.search(index='indicators-*')
s = s.params(size=limit, timeout=timeout)
if sort:
s = s.sort(sort)
q_filters = {}
for f in VALID_FILTERS:
if filters.get(f):
q_filters[f] = filters[f]
if q_filters.get('indicator'):
try:
itype = resolve_itype(q_filters['indicator'])
if itype == 'ipv4':
if PYVERSION == 2:
q_filters['indicator'] = unicode(
q_filters['indicator'])
ip = ipaddress.IPv4Network(q_filters['indicator'])
mask = ip.prefixlen
if mask < 8:
raise InvalidSearch(
'prefix needs to be greater than or equal to 8')
start = str(ip.network_address)
end = str(ip.broadcast_address)
s = s.filter('range',
indicator_ipv4={
'gte': start,
'lte': end
})
elif itype is 'ipv6':
if PYVERSION == 2:
q_filters['indicator'] = unicode(
q_filters['indicator'])
ip = ipaddress.IPv6Network(q_filters['indicator'])
mask = ip.prefixlen
if mask < 32:
raise InvalidSearch(
'prefix needs to be greater than or equal to 32')
start = binascii.b2a_hex(
socket.inet_pton(
socket.AF_INET6,
str(ip.network_address))).decode('utf-8')
end = binascii.b2a_hex(
socket.inet_pton(
socket.AF_INET6,
str(ip.broadcast_address))).decode('utf-8')
s = s.filter('range',
indicator_ipv6={
'gte': start,
'lte': end
})
elif itype in ('email', 'url', 'fqdn'):
s = s.filter('term', indicator=q_filters['indicator'])
except InvalidIndicator:
s = s.query("match", message=q_filters['indicator'])
del q_filters['indicator']
for f in q_filters:
kwargs = {f: q_filters[f]}
s = s.filter('term', **kwargs)
try:
rv = s.execute()
except elasticsearch.exceptions.RequestError as e:
self.logger.error(e)
return []
if raw:
try:
return rv.hits.hits
except KeyError:
return []
else:
try:
data = []
for x in rv.hits.hits:
if x['_source'].get('message'):
x['_source']['message'] = b64encode(
x['_source']['message'].encode('utf-8'))
data.append(x['_source'])
return data
except KeyError as e:
self.logger.error(e)
return []
