def esSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() -
timedelta(minutes=options.aggregationminutes))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
# search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qDate]))
q = q.search()
qagg = pyes.aggs.TermsAgg(name='category', field='category')
q.agg.add(qagg)
results = es.search(query=q, indices=['events'])
mozdefstats = dict(utctimestamp=toUTC(datetime.now()).isoformat())
mozdefstats['summary'] = 'Aggregated category counts'
mozdefstats['processid'] = os.getpid()
mozdefstats['processname'] = sys.argv[0]
mozdefstats['details'] = dict(counts=list())
for bucket in results.aggs['category']['buckets']:
entry = dict()
entry[bucket['key']] = bucket['doc_count']
mozdefstats['details']['counts'].append(entry)
return mozdefstats
except pyes.exceptions.NoServerAvailable:
logger.error(
'Elastic Search server could not be reached, check network connectivity'
)
def esCloudTrailSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(hours=160))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
#search for actions within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qcloud = pyes.TermFilter('_type', 'cloudtrail')
qEvents = pyes.TermsFilter(
'eventName', ['runinstances', 'stopinstances', 'startinstances'])
qalerted = pyes.ExistsFilter('alerttimestamp')
results = es.search(pyes.ConstantScoreQuery(
pyes.BoolFilter(must=[qcloud, qDate, qEvents],
must_not=[qalerted])),
indices='events')
#uncomment for debugging to recreate alerts for events that already have an alerttimestamp
#results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents])))
return (results._search_raw()['hits']['hits'])
except pyes.exceptions.NoServerAvailable:
logger.error(
'Elastic Search server could not be reached, check network connectivity'
)
def searchESForBROAttackers(es, threshold):
begindateUTC = toUTC(datetime.now() - timedelta(hours=2))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
qBro = pyes.QueryFilter(pyes.MatchQuery('category', 'bro_notice',
'phrase'))
qErr = pyes.QueryFilter(
pyes.MatchQuery('details.note',
'MozillaHTTPErrors::Excessive_HTTP_Errors_Attacker',
'phrase'))
q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qBro, qErr]))
results = es.search(q, size=1000, indices='events,events-previous')
# grab results as native es results to avoid pyes iteration bug
# and get easier access to es metadata fields
rawresults = results._search_raw()['hits']['hits']
# Hit count is buried in the 'sub' field
# as: 'sub': u'6 in 1.0 hr, eps: 0'
# cull the records for hitcounts over the threshold before returning
attackers = list()
for r in rawresults:
hitcount = int(r['_source']['details']['sub'].split()[0])
if hitcount > threshold:
attackers.append(r)
return attackers
def filtersManual(self, date_timedelta, must=[], should=[], must_not=[]):
"""
Configure filters manually
date_timedelta is a dict in timedelta format
see https://docs.python.org/2/library/datetime.html#timedelta-objects
must, should and must_not are pyes filter objects lists
see http://pyes.readthedocs.org/en/latest/references/pyes.filters.html
"""
self.begindateUTC = toUTC(datetime.now() - timedelta(**date_timedelta))
self.enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',
from_value=self.begindateUTC, to_value=self.enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
#Don't fire on already alerted events
if pyes.ExistsFilter('alerttimestamp') not in must_not:
must_not.append(pyes.ExistsFilter('alerttimestamp'))
must.append(qDate)
q.filters.append(pyes.BoolFilter(
must=must,
should=should,
must_not=must_not))
self.filter = q
def esSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.aggregationminutes))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
# search for aggregated event stats summaries within the date range
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q = pyes.FilteredQuery(q,pyes.BoolFilter(must=[qDate,
pyes.TermFilter('_type', 'mozdefstats')]))
results=es.search(query=q,size=100,indices=['events','events-previous'])
#avoid pyes iteration bug
rawresults = results._search_raw()
#examine the results
#for each details.counts, append the count
#as a list to the stats dict
stats=dict()
for r in rawresults['hits']['hits']:
for i in r['_source']['details']['counts']:
#print(i.values()[0])
if i.keys()[0] not in stats.keys():
stats[i.keys()[0]]=list()
stats[i.keys()[0]].append(i.values()[0])
# make a dictionairy of user-defined
# aggregation threshold percentages
aggregationthresholds = dict(zip(options.aggregations,
options.aggregationthresholds))
#for our running history of counts per category
#do some simple stats math to see if we
#should alert on anything
for s in stats:
alert = False
smean=round(numpy.mean(stats[s]))
sstd=round(numpy.std(stats[s]))
stat = round((sstd/smean)*100, 2)
if s in aggregationthresholds.keys():
if stat > aggregationthresholds[s]:
alert = True
elif stat > options.defaultthreshold:
alert = True
if alert:
print('{0} {1}%: \n\t\t{2} \n\t\t{3} \n\t\t{4}'.format(
s,
stat,
stats[s],
smean,
sstd
)
)
except pyes.exceptions.NoServerAvailable:
logger.error('Elastic Search server could not be reached, check network connectivity')
def esLdapResults(begindateUTC=None, enddateUTC=None):
'''an ES query/facet to count success/failed logins'''
resultsList = list()
if begindateUTC is None:
begindateUTC = datetime.now() - timedelta(hours=1)
begindateUTC = toUTC(begindateUTC)
if enddateUTC is None:
enddateUTC = datetime.now()
enddateUTC = toUTC(enddateUTC)
try:
es = pyes.ES((list('{0}'.format(s) for s in options.esservers)))
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',
from_value=begindateUTC, to_value=enddateUTC))
q = pyes.MatchAllQuery()
q = pyes.FilteredQuery(q, qDate)
q = pyes.FilteredQuery(q, pyes.TermFilter('tags', 'ldap'))
q = pyes.FilteredQuery(q,
pyes.TermFilter('details.result', 'ldap_invalid_credentials'))
q2 = q.search()
q2.facet.add_term_facet('details.result')
q2.facet.add_term_facet('details.dn', size=20)
results = es.search(q2, indices='events')
stoplist = ('o', 'mozilla', 'dc', 'com', 'mozilla.com',
'mozillafoundation.org', 'org')
for t in results.facets['details.dn'].terms:
if t['term'] in stoplist:
continue
#print(t['term'])
failures = 0
success = 0
dn = t['term']
#re-query with the terms of the details.dn
qt = pyes.MatchAllQuery()
qt = pyes.FilteredQuery(qt, qDate)
qt = pyes.FilteredQuery(qt, pyes.TermFilter('tags', 'ldap'))
qt = pyes.FilteredQuery(qt,
pyes.TermFilter('details.dn', t['term']))
qt2 = qt.search()
qt2.facet.add_term_facet('details.result')
results = es.search(qt2)
#sys.stdout.write('{0}\n'.format(results.facets['details.result'].terms))
for t in results.facets['details.result'].terms:
#print(t['term'],t['count'])
if t['term'] == 'ldap_success':
success = t['count']
if t['term'] == 'ldap_invalid_credentials':
failures = t['count']
resultsList.append(dict(dn=dn, failures=failures,
success=success, begin=begindateUTC.isoformat(),
end=enddateUTC.isoformat()))
return(json.dumps(resultsList))
except pyes.exceptions.NoServerAvailable:
sys.stderr.write('Elastic Search server could not be reached, check network connectivity\n')
def make_query():
qrange = pyes.RangeQuery(pyes.ESRange('utctimestamp', start_date, end_date))
qterm = pyes.TermQuery('_type', 'alert')
q = pyes.BoolQuery(must=[qrange, qterm])
results = esconn.search(q, indices=indexname)
ret = []
for x in results:
na = Alert()
if na.parse_result(x): ret.append(na)
return ret
def getESAlerts(es):
begindateUTC = toUTC(datetime.now() - timedelta(minutes=50))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'alert')
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(pyes.BoolFilter(must=[qDate, qType]))
results = es.search(q, size=10000, indices='alerts')
# return raw search to avoid pyes iteration bug
return results._search_raw()
def getFrontendStats(es):
begindateUTC = toUTC(datetime.now() - timedelta(minutes=1))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'mozdefhealth')
qMozdef = pyes.TermsFilter('category', ['mozdef'])
pyesresults = es.search(pyes.ConstantScoreQuery(
pyes.BoolFilter(must=[qType, qDate, qMozdef])),
indices='events')
return pyesresults._search_raw()['hits']['hits']
def esBroXSSEvents():
begindateUTC = toUTC(datetime.now() - timedelta(minutes=30))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermFilter("category", "broxsslog")
qalerted = pyes.ExistsFilter('alerttimestamp')
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(must=[qType, qDate, qEvents,
pyes.ExistsFilter('uri')],
must_not=[qalerted]))
return q
def esSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(minutes=60))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
#search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermsFilter('category', ['brointel'])
qalerted = pyes.ExistsFilter('alerttimestamp')
qdetails = pyes.ExistsFilter('details')
qindicator = pyes.ExistsFilter('seenindicator')
pyesresults = es.search(pyes.ConstantScoreQuery(
pyes.BoolFilter(must=[qType, qDate, qEvents, qdetails, qindicator],
must_not=[qalerted])),
size=1000)
#uncomment for debugging to recreate alerts for events that already have an alerttimestamp
#results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents])))
#logger.debug(results.count())
#correlate any matches by the seenindicator field.
#make a simple list of indicator values that can be counted/summarized by Counter
resultsIndicators = list()
#bug in pyes..capture results as raw list or it mutates after first access:
#copy the hits.hits list as our resusts, which is the same as the official elastic search library returns.
results = pyesresults._search_raw()['hits']['hits']
for r in results:
resultsIndicators.append(r['_source']['details']['seenindicator'])
#use the list of tuples ('indicator',count) to create a dictionary with:
#indicator,count,es records
#and add it to a list to return.
indicatorList = list()
for i in Counter(resultsIndicators).most_common():
idict = dict(indicator=i[0], count=i[1], events=[])
for r in results:
if r['_source']['details']['seenindicator'].encode(
'ascii', 'ignore') == i[0]:
idict['events'].append(r)
indicatorList.append(idict)
return indicatorList
except pyes.exceptions.NoServerAvailable:
logger.error(
'Elastic Search server could not be reached, check network connectivity'
)
def esBroIntelEvents():
begindateUTC = toUTC(datetime.now() - timedelta(minutes=30))
enddateUTC = toUTC(datetime.now())
#search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermsFilter('category', ['brointel'])
qalerted = pyes.ExistsFilter('alerttimestamp')
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(
must=[qType, qDate, qEvents,
pyes.ExistsFilter('seenindicator')],
must_not=[qalerted]))
return q
def search_range_time(field, start_date, date_range, index_name, index_type):
if type(date_range) == type(-1) and date_range != -1:
#start_da = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M:%SZ").date()
start_da = datetime.datetime.strptime(start_date, "%Y-%m-%d").date()
end_date = (start_da +
datetime.timedelta(days=date_range)).strftime('%Y-%m-%d')
must = pyes.RangeQuery(
pyes.ESRange(field, from_value=start_date, to_value=end_date))
query = pyes.BoolQuery(must=must)
dd = [
i for i in CONN_ES.search(
query=query, indices=index_name, doc_types=index_type)
]
return dd
else:
raise
def esSearch(es, macassignments=None, begindateUTC=None, enddateUTC=None):
'''
Search ES for an event that ties a username to a mac address
This example searches for junos wifi correlations on authentication success
Expecting an event like: user: [email protected]; mac: 5c:f9:38:b1:de:cf; author reason: roamed session; ssid: ANSSID; AP 46/2\n
'''
usermacre=re.compile(r'''user: (?P<username>.*?); mac: (?P<macaddress>.*?); ''',re.IGNORECASE)
correlations={} # list of dicts to populate hits we find
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.correlationminutes))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
# search for events within the date range
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q=pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(pyes.BoolFilter(must=[
qDate,
pyes.TermFilter("program","AUTHORIZATION-SUCCESS")
],
must_not=[
pyes.QueryFilter(
pyes.MatchQuery("summary","last-resort","phrase")
)
]))
results = es.search(q, size=10000, indices=['events', 'events-previous'])
rawresults=results._search_raw()
for r in rawresults['hits']['hits']:
fields = re.search(usermacre,r['_source']['summary'])
if fields:
if '{0} {1}'.format(fields.group('username'),fields.group('macaddress')) not in correlations.keys():
if fields.group('macaddress')[0:8].lower() in macassignments.keys():
entity=macassignments[fields.group('macaddress')[0:8].lower()]
else:
entity='unknown'
correlations['{0} {1}'.format(fields.group('username'),fields.group('macaddress'))]=dict(username=fields.group('username'),
macaddress=fields.group('macaddress'),
entity=entity,
utctimestamp=r['_source']['utctimestamp'])
return correlations
except pyes.exceptions.NoServerAvailable:
logger.error('Elastic Search server could not be reached, check network connectivity')
def searchForSSHKeys(es):
begindateUTC = toUTC(datetime.now() - timedelta(minutes=5))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermFilter("program", "sshd")
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(must=[
qType, qDate, qEvents,
pyes.QueryFilter(
pyes.MatchQuery("summary",
"found matching key accepted publickey",
"boolean"))
]))
results = es.search(q, size=10000, indices='events')
# return raw search to avoid pyes iteration bug
return results._search_raw()
def search_err(self, program, receiver):
start = "%s CST" % (datetime.astimezonenow() - timedelta(
hours=self.config['RANGE'])).daysstrftime('%Y/%m/%d %H:%M:%S')
end = "%s CST" % datetime.astimezonenow().strftime('%Y/%m/%d %H:%M:%S')
index = "%s_%s" % (program, datetime.astimezonenow().strftime('%Y-%m'))
q = pyes.RangeQuery(
pyes.ESRange('datetime', from_value=start, to_value=end))
q = pyes.FilteredQuery(q, pyes.TermQuery("level", "EROR"))
conn = pyes.ES(self.config['ES'])
results = conn.search(indices=index, query=q)
num = len(results)
if num == 0:
return
_content = ""
for r in results:
_content += "%s, %s<br/>" % (r['message'], r['datetime'])
title = self.config['MAIL']['MAIL_TITLE'] % program
content = "<b>Program:%s</b></b><br/>From:%s<br/>To:%s<br/><b>Total Num: %d</b><br/><br/><b>Content are:</b><br/>%s" % (
program, start, end, num, _content)
self.sendmail(receiver, title, content)
def query_submissions(q='',
fq=None,
sort='score desc',
start=0,
docs=10,
date=None,
facets=None):
(sort_field, sort_order) = sort.split(' ')
if sort_field == 'score':
sort_field = '_score'
sort = {sort_field: {'order': sort_order}}
query = pyes.query.BoolQuery()
query.add_must(pyes.StringQuery(q, default_operator="AND"))
rest = True
x = 0
result = []
while rest:
y = fq.find(":", x)
if y == -1:
break
temp = fq[x:y]
x = y + 1
if fq[x:x + 5] == """:
y = fq.find(""", x + 5)
if y == -1:
break
result.append((temp, fq[x + 5:y]))
x = y + 6
if x > len(fq):
break
else:
y = fq.find(";", x)
if y == -1:
result.append((temp, fq[x:len(fq)]))
break
else:
result.append((temp, fq[x:y]))
x = y + 1
for sfq in result:
if sfq[0] == 'date':
(year, month) = sfq[1].split('-')
date_start = datetime.datetime(int(year), int(month), 1)
date_end = date_start + dateutil.relativedelta.relativedelta(
months=+1, seconds=-1)
query.add_must(
pyes.RangeQuery(
qrange=pyes.ESRange('date', date_start, date_end)))
else:
query.add_must(pyes.TermQuery(field=sfq[0], value=sfq[1]))
search = pyes.query.Search(query=query,
fields=[''],
start=start,
size=docs,
sort=sort)
search.facet.add_term_facet('type')
search.facet.add_term_facet('rs')
search.facet.add_term_facet('committee')
search.facet.add_date_facet(field='date', name='date', interval='month')
es = pyes.ES(app.config['ES_HOST'] + ':' + str(app.config['ES_PORT']))
es.default_indices = [app.config['ES_INDEX_NAME_PREFIX'] + '-latest']
es.refresh()
result = es.search(search, model=lambda x, y: y)
ret = {
'numhits': result.total,
'maxscore': result.max_score,
'result': [],
'facets': {}
}
if result.max_score is not None:
ret['maxscore'] = result.max_score
for key in result.facets:
ret['facets'][key] = {}
if result.facets[key]['_type'] == 'date_histogram':
for subval in result.facets[key]['entries']:
ret['facets'][key][datetime.datetime.fromtimestamp(
int(subval['time']) /
1000).strftime('%Y-%m')] = subval['count']
if result.facets[key]['_type'] == 'terms':
for subval in result.facets[key]['terms']:
ret['facets'][key][subval['term']] = subval['count']
for r in result:
ret['result'].append({'_id': str(r['_id']), 'score': r['_score']})
return ret
