def filtersManual(self, date_timedelta, must=[], should=[], must_not=[]):
"""
Configure filters manually
date_timedelta is a dict in timedelta format
see https://docs.python.org/2/library/datetime.html#timedelta-objects
must, should and must_not are pyes filter objects lists
see http://pyes.readthedocs.org/en/latest/references/pyes.filters.html
"""
self.begindateUTC = toUTC(datetime.now() - timedelta(**date_timedelta))
self.enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',
from_value=self.begindateUTC, to_value=self.enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
#Don't fire on already alerted events
if pyes.ExistsFilter('alerttimestamp') not in must_not:
must_not.append(pyes.ExistsFilter('alerttimestamp'))
must.append(qDate)
q.filters.append(pyes.BoolFilter(
must=must,
should=should,
must_not=must_not))
self.filter = q
def esBroXSSEvents():
begindateUTC = toUTC(datetime.now() - timedelta(minutes=30))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermFilter("category", "broxsslog")
qalerted = pyes.ExistsFilter('alerttimestamp')
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(must=[qType, qDate, qEvents,
pyes.ExistsFilter('uri')],
must_not=[qalerted]))
return q
def esSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(minutes=60))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
#search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermsFilter('category', ['brointel'])
qalerted = pyes.ExistsFilter('alerttimestamp')
qdetails = pyes.ExistsFilter('details')
qindicator = pyes.ExistsFilter('seenindicator')
pyesresults = es.search(pyes.ConstantScoreQuery(
pyes.BoolFilter(must=[qType, qDate, qEvents, qdetails, qindicator],
must_not=[qalerted])),
size=1000)
#uncomment for debugging to recreate alerts for events that already have an alerttimestamp
#results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents])))
#logger.debug(results.count())
#correlate any matches by the seenindicator field.
#make a simple list of indicator values that can be counted/summarized by Counter
resultsIndicators = list()
#bug in pyes..capture results as raw list or it mutates after first access:
#copy the hits.hits list as our resusts, which is the same as the official elastic search library returns.
results = pyesresults._search_raw()['hits']['hits']
for r in results:
resultsIndicators.append(r['_source']['details']['seenindicator'])
#use the list of tuples ('indicator',count) to create a dictionary with:
#indicator,count,es records
#and add it to a list to return.
indicatorList = list()
for i in Counter(resultsIndicators).most_common():
idict = dict(indicator=i[0], count=i[1], events=[])
for r in results:
if r['_source']['details']['seenindicator'].encode(
'ascii', 'ignore') == i[0]:
idict['events'].append(r)
indicatorList.append(idict)
return indicatorList
except pyes.exceptions.NoServerAvailable:
logger.error(
'Elastic Search server could not be reached, check network connectivity'
)
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'event'),
pyes.TermFilter('eventsource', 'systemslogs'),
pyes.ExistsFilter('details.sourceipaddress'),
pyes.QueryFilter(pyes.MatchQuery('summary', 'failed', 'phrase')),
pyes.TermFilter('program', 'sshd'),
pyes.QueryFilter(
pyes.MatchQuery('summary', 'login ldap_count_entries',
'boolean'))
]
must_not = [
pyes.QueryFilter(
pyes.MatchQuery('summary', '10.22.8.128', 'phrase')),
pyes.QueryFilter(pyes.MatchQuery('summary', '10.8.75.35',
'phrase')),
pyes.QueryFilter(
pyes.MatchQuery('summary', '208.118.237.', 'phrase'))
]
self.filtersManual(date_timedelta, must=must, must_not=must_not)
# Search aggregations on field 'sourceipaddress', keep 50 samples of events at most
self.searchEventsAggreg('sourceipaddress', samplesLimit=50)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=10)
def esBroIntelEvents():
begindateUTC = toUTC(datetime.now() - timedelta(minutes=30))
enddateUTC = toUTC(datetime.now())
#search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermsFilter('category', ['brointel'])
qalerted = pyes.ExistsFilter('alerttimestamp')
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(
must=[qType, qDate, qEvents,
pyes.ExistsFilter('seenindicator')],
must_not=[qalerted]))
return q
def esCloudTrailSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(hours=160))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
#search for actions within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qcloud = pyes.TermFilter('_type', 'cloudtrail')
qEvents = pyes.TermsFilter(
'eventName', ['runinstances', 'stopinstances', 'startinstances'])
qalerted = pyes.ExistsFilter('alerttimestamp')
results = es.search(pyes.ConstantScoreQuery(
pyes.BoolFilter(must=[qcloud, qDate, qEvents],
must_not=[qalerted])),
indices='events')
#uncomment for debugging to recreate alerts for events that already have an alerttimestamp
#results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents])))
return (results._search_raw()['hits']['hits'])
except pyes.exceptions.NoServerAvailable:
logger.error(
'Elastic Search server could not be reached, check network connectivity'
)
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.ExistsFilter('details.hostname'),
pyes.QueryFilter(pyes.MatchQuery('summary','DuoAPI contact failed','phrase'))
]
self.filtersManual(date_timedelta, must=must)
# Search aggregations on field 'sourceipaddress', keep X samples of events at most
self.searchEventsAggreg('hostname', samplesLimit=10)
# alert when >= X matching events in an aggregation
# in this case, always
self.walkAggregations(threshold=1)
def main(self):
# look for events in last 30 mins
date_timedelta = dict(minutes=30)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'event'),
pyes.TermFilter('category', 'brointel'),
pyes.ExistsFilter('seenindicator')
]
self.filtersManual(date_timedelta, must=must)
# Search aggregations on field 'seenindicator', keep 50 samples of events at most
self.searchEventsAggregated('details.seenindicator', samplesLimit=50)
# alert when >= 5 matching events in an aggregation
self.walkAggregations(threshold=5)
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'bro'),
pyes.TermFilter('eventsource', 'nsm'),
pyes.TermFilter('category', 'bronotice'),
pyes.ExistsFilter('details.sourceipaddress'),
pyes.QueryFilter(pyes.MatchQuery('details.note','SSH::Password_Guessing','phrase')),
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'bro'),
pyes.TermFilter('eventsource', 'nsm'),
pyes.TermFilter('category', 'brointel'),
pyes.TermFilter('details.sources', 'abuse.ch SSLBL'),
pyes.ExistsFilter('details.sourceipaddress')
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
# look for events in last X mins
date_timedelta = dict(minutes=2)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'bro'),
pyes.TermFilter('eventsource', 'nsm'),
pyes.TermFilter('category', 'brointel'),
pyes.ExistsFilter('seenindicator'),
pyes.QueryFilter(pyes.MatchQuery('hostname', 'sensor1 sensor2 sensor3', 'boolean'))
]
self.filtersManual(date_timedelta, must=must)
# Search aggregations on field 'seenindicator', keep X samples of events at most
self.searchEventsAggregated('details.seenindicator', samplesLimit=10)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=10)
def filtersFromKibanaDash(self, fp, date_timedelta):
"""
Import filters from a kibana dashboard
fp is the file path of the json file
date_timedelta is a dict in timedelta format
see https://docs.python.org/2/library/datetime.html#timedelta-objects
"""
f = open(fp)
data = json.load(f)
must = []
should = []
must_not = []
for filtid in data['services']['filter']['list'].keys():
filt = data['services']['filter']['list'][filtid]
if filt['active'] and 'query' in filt.keys():
value = filt['query'].split(':')[-1]
fieldname = filt['query'].split(':')[0].split('.')[-1]
# self.log.info(fieldname)
# self.log.info(value)
# field: fieldname
# query: value
if 'field' in filt.keys():
fieldname = filt['field']
value = filt['query']
if '\"' in value:
value = value.split('\"')[1]
pyesfilt = pyes.QueryFilter(
pyes.MatchQuery(fieldname, value, 'phrase'))
else:
pyesfilt = pyes.TermFilter(fieldname, value)
else:
# _exists_:field
if filt['query'].startswith('_exists_:'):
pyesfilt = pyes.ExistsFilter(value.split('.')[-1])
# self.log.info('exists %s' % value.split('.')[-1])
# _missing_:field
elif filt['query'].startswith('_missing_:'):
pyesfilt = pyes.filters.MissingFilter(
value.split('.')[-1])
# self.log.info('missing %s' % value.split('.')[-1])
# field:"value"
elif '\"' in value:
value = value.split('\"')[1]
pyesfilt = pyes.QueryFilter(
pyes.MatchQuery(fieldname, value, 'phrase'))
# self.log.info("phrase %s %s" % (fieldname, value))
# field:(value1 value2 value3)
elif '(' in value and ')' in value:
value = value.split('(')[1]
value = value.split('(')[0]
pyesfilt = pyes.QueryFilter(
pyes.MatchQuery(fieldname, value, "boolean"))
# field:value
else:
pyesfilt = pyes.TermFilter(fieldname, value)
# self.log.info("terms %s %s" % (fieldname, value))
if filt['mandate'] == 'must':
must.append(pyesfilt)
elif filt['mandate'] == 'either':
should.append(pyesfilt)
elif filt['mandate'] == 'mustNot':
must_not.append(pyesfilt)
# self.log.info(must)
f.close()
self.filtersManual(date_timedelta,
must=must,
should=should,
must_not=must_not)
