def get_iplists(self):
ip_lists = {}
# Get all IPs except from tor
q = {
'query': {
'bool': {
'must_not': [{
'match': {
'iplist.name': 'tor'
}
}]
}
}
}
res = rawSearch(q, index='redelk-iplist-*')
if not res:
return (ip_lists)
for ipdoc in res['hits']['hits']:
ip = getValue('_source.iplist.ip', ipdoc)
iplist_name = getValue('_source.iplist.name', ipdoc)
# Already one IP found in this list, adding it
if iplist_name in ip_lists:
ip_lists[iplist_name].append(ip)
# First IP for this IP list, creating the array
else:
ip_lists[iplist_name] = [ip]
return (ip_lists)
def get_redirtraffic(self):
# Get all redirtraffic before 'now' that were not processed by previous run of the module
q = {
'sort': [{
'@timestamp': {
'order': 'desc'
}
}],
'query': {
'bool': {
'filter': [{
'range': {
'@timestamp': {
'lte': self.now.isoformat()
}
}
}],
'must_not': [{
'match': {
'tags': info['submodule']
}
}]
}
}
}
res = rawSearch(q, index='redirtraffic-*')
self.logger.debug(res)
if res is None:
return ([])
else:
return (res['hits']['hits'])
def get_last_sync(self):
# Get greynoise data from ES if less than 1 day old
q = {
"size": 1,
"sort": [{
"@timestamp": {
"order": "desc"
}
}],
"query": {
"bool": {
"filter": [{
"term": {
"iplist.name": "tor"
}
}]
}
}
}
res = rawSearch(q, index='redelk-*')
self.logger.debug(res)
# Return the latest hit or False if not found
if res and len(res['hits']['hits']) > 0:
dt_str = getValue('_source.@timestamp', res['hits']['hits'][0])
dt = datetime.datetime.strptime(dt_str, '%Y-%m-%dT%H:%M:%S.%f')
return (dt)
else:
return (datetime.datetime.fromtimestamp(0))
def get_alarmed_ips(self):
query = {
'sort': [{'@timestamp': {'order': 'desc'}}],
'query': {
'bool': {
'filter': [
{
'range': {
'@timestamp': {
'gte': 'now-1y'
}
}
},
{'match': {'tags': info['submodule']}}
]
}
}
}
res = rawSearch(query, index='redirtraffic-*')
if res is None:
alarmed = []
else:
alarmed = res['hits']['hits']
# Created a dict grouped by IP address (from source.ip)
ips = {}
for al in alarmed:
ip = getValue('_source.source.ip', al)
if ip in ips:
ips[ip].append(al)
else:
ips[ip] = [al]
return(ips)
def get_es_tor_exitnodes(self):
q = {'query': {'bool': {'filter': {'term': {'iplist.name': 'tor'}}}}}
res = rawSearch(q, index='redelk-*')
if not res:
return []
iplist = []
for ipdoc in res['hits']['hits']:
ip = getValue('_source.iplist.ip', ipdoc)
iplist.append(ip)
return (iplist)
def alarm_check(self, alarmed_ips):
# This check queries for IP's that aren't listed in any iplist* but do talk to c2* paths on redirectors
query = {
'sort': [{'@timestamp': {'order': 'desc'}}],
'query': {
'bool': {
'filter': [
{'match': {'tags': 'enrich_iplists'}}
],
'must': {
'query_string': {
'fields': ['redir.backend.name'],
'query': 'c2-*'
}
},
'must_not': [{
'query_string': {
'fields': ['tags'],
'query': 'iplist_*'
}
},
{'match': {'tags': info['submodule']}}
]
}
}
}
res = rawSearch(query, index='redirtraffic-*')
if res is None:
notEnriched = []
else:
notEnriched = res['hits']['hits']
# Created a dict grouped by IP address (from source.ip)
ips = {}
for ne in notEnriched:
ip = getValue('_source.source.ip', ne)
if ip in ips:
ips[ip].append(ne)
else:
ips[ip] = [ne]
hits = []
# Now we check if the IPs have already been alarmed in the past timeframe defined in the config
for ip in ips:
# Not alarmed yet, process it
if ip not in alarmed_ips:
hits += ips[ip]
# Return the array of new IP documents to be alarmed
return(hits)
def enrich_tor(self, iplist):
# Get all lines in redirtraffic that have not been enriched with 'enrich_iplist' or 'enrich_tor'
# Filter documents that were before the last run time of enrich_iplist (to avoid race condition)
iplist_lastrun = getLastRun('enrich_iplists')
query = {
'sort': [{
'@timestamp': {
'order': 'desc'
}
}],
'query': {
'bool': {
'filter': [{
'range': {
'@timestamp': {
'lte': iplist_lastrun.isoformat()
}
}
}],
'must_not': [{
'match': {
'tags': info['submodule']
}
}]
}
}
}
res = rawSearch(query, index='redirtraffic-*')
if res is None:
notEnriched = []
else:
notEnriched = res['hits']['hits']
# For each IP, check if it is in tor exit node data
hits = []
for ne in notEnriched:
ip = getValue('_source.source.ip', ne)
if ip in iplist:
hits.append(ne)
return (hits)
def get_last_es_data(self, ip):
# Get greynoise data from ES if less than 1 day old
q = {
"size": 1,
"sort": [{
"@timestamp": {
"order": "desc"
}
}],
"query": {
"bool": {
"filter": [{
"range": {
"greynoise.query_timestamp": {
"gte": "now-%ss" % self.cache,
"lte": "now"
}
}
}, {
"term": {
"tags": "enrich_greynoise"
}
}, {
"term": {
"host.ip": ip
}
}]
}
}
}
res = rawSearch(q, index='redirtraffic-*')
self.logger.debug(res)
# Return the latest hit or False if not found
if res and len(res['hits']['hits']) > 0:
return (res['hits']['hits'][0])
else:
return (False)
def enrich_greynoise(self):
# Get all lines in redirtraffic that have not been enriched with 'enrich_greynoise'
# Filter documents that were before the last run time of enrich_iplist (to avoid race condition)
iplist_lastrun = getLastRun('enrich_iplists')
query = {
'sort': [{
'@timestamp': {
'order': 'desc'
}
}],
'query': {
'bool': {
'filter': [{
'range': {
'@timestamp': {
'lte': iplist_lastrun.isoformat()
}
}
}],
'must_not': [{
'match': {
'tags': info['submodule']
}
}]
}
}
}
res = rawSearch(query, index='redirtraffic-*')
if res is None:
notEnriched = []
else:
notEnriched = res['hits']['hits']
# Created a dict grouped by IP address (from source.ip)
ips = {}
for ne in notEnriched:
ip = getValue('_source.source.ip', ne)
if ip in ips:
ips[ip].append(ne)
else:
ips[ip] = [ne]
hits = []
# For each IP, get the greynoise data
for ip in ips:
# Get data from redirtraffic if within interval
lastESData = self.get_last_es_data(ip)
if not lastESData:
greynoiseData = self.get_greynoise_data(ip)
else:
greynoiseData = getValue('_source.greynoise', lastESData)
# If no greynoise data found, skip the IP
if not greynoiseData:
continue
for doc in ips[ip]:
# Fields to copy: greynoise.*
res = self.add_greynoise_data(doc, greynoiseData)
if res:
hits.append(res)
return (hits)