def getFrontendStats(es):
begindateUTC = toUTC(datetime.now() - timedelta(minutes=15), options.defaulttimezone)
enddateUTC = toUTC(datetime.now(), options.defaulttimezone)
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',
from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'mozdefhealth')
qMozdef = pyes.TermsFilter('category', ['mozdef'])
qLatest = pyes.TermsFilter('tags', ['latest'])
pyesresults = es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(
must=[qDate, qType, qLatest, qMozdef])),
indices='events')
return pyesresults._search_raw()['hits']['hits']
def esCloudTrailSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(hours=160))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
#search for actions within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qcloud = pyes.TermFilter('_type', 'cloudtrail')
qEvents = pyes.TermsFilter(
'eventName', ['runinstances', 'stopinstances', 'startinstances'])
qalerted = pyes.ExistsFilter('alerttimestamp')
results = es.search(pyes.ConstantScoreQuery(
pyes.BoolFilter(must=[qcloud, qDate, qEvents],
must_not=[qalerted])),
indices='events')
#uncomment for debugging to recreate alerts for events that already have an alerttimestamp
#results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents])))
return (results._search_raw()['hits']['hits'])
except pyes.exceptions.NoServerAvailable:
logger.error(
'Elastic Search server could not be reached, check network connectivity'
)
def main(self):
# look for events in last x hours
date_timedelta = dict(hours=1)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'cloudtrail'),
pyes.TermsFilter('eventName',['runinstances','stopinstances','startinstances'])
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def esSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(minutes=60))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
#search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermsFilter('category', ['brointel'])
qalerted = pyes.ExistsFilter('alerttimestamp')
qdetails = pyes.ExistsFilter('details')
qindicator = pyes.ExistsFilter('seenindicator')
pyesresults = es.search(pyes.ConstantScoreQuery(
pyes.BoolFilter(must=[qType, qDate, qEvents, qdetails, qindicator],
must_not=[qalerted])),
size=1000)
#uncomment for debugging to recreate alerts for events that already have an alerttimestamp
#results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents])))
#logger.debug(results.count())
#correlate any matches by the seenindicator field.
#make a simple list of indicator values that can be counted/summarized by Counter
resultsIndicators = list()
#bug in pyes..capture results as raw list or it mutates after first access:
#copy the hits.hits list as our resusts, which is the same as the official elastic search library returns.
results = pyesresults._search_raw()['hits']['hits']
for r in results:
resultsIndicators.append(r['_source']['details']['seenindicator'])
#use the list of tuples ('indicator',count) to create a dictionary with:
#indicator,count,es records
#and add it to a list to return.
indicatorList = list()
for i in Counter(resultsIndicators).most_common():
idict = dict(indicator=i[0], count=i[1], events=[])
for r in results:
if r['_source']['details']['seenindicator'].encode(
'ascii', 'ignore') == i[0]:
idict['events'].append(r)
indicatorList.append(idict)
return indicatorList
except pyes.exceptions.NoServerAvailable:
logger.error(
'Elastic Search server could not be reached, check network connectivity'
)
def esBroIntelEvents():
begindateUTC = toUTC(datetime.now() - timedelta(minutes=30))
enddateUTC = toUTC(datetime.now())
#search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermsFilter('category', ['brointel'])
qalerted = pyes.ExistsFilter('alerttimestamp')
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(
must=[qType, qDate, qEvents,
pyes.ExistsFilter('seenindicator')],
must_not=[qalerted]))
return q