def searchESForBROAttackers(es, threshold):
begindateUTC = toUTC(datetime.now() - timedelta(hours=2))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
qBro = pyes.QueryFilter(pyes.MatchQuery('category', 'bro_notice',
'phrase'))
qErr = pyes.QueryFilter(
pyes.MatchQuery('details.note',
'MozillaHTTPErrors::Excessive_HTTP_Errors_Attacker',
'phrase'))
q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qBro, qErr]))
results = es.search(q, size=1000, indices='events,events-previous')
# grab results as native es results to avoid pyes iteration bug
# and get easier access to es metadata fields
rawresults = results._search_raw()['hits']['hits']
# Hit count is buried in the 'sub' field
# as: 'sub': u'6 in 1.0 hr, eps: 0'
# cull the records for hitcounts over the threshold before returning
attackers = list()
for r in rawresults:
hitcount = int(r['_source']['details']['sub'].split()[0])
if hitcount > threshold:
attackers.append(r)
return attackers
def esSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() -
timedelta(minutes=options.aggregationminutes))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
# search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qDate]))
q = q.search()
qagg = pyes.aggs.TermsAgg(name='category', field='category')
q.agg.add(qagg)
results = es.search(query=q, indices=['events'])
mozdefstats = dict(utctimestamp=toUTC(datetime.now()).isoformat())
mozdefstats['summary'] = 'Aggregated category counts'
mozdefstats['processid'] = os.getpid()
mozdefstats['processname'] = sys.argv[0]
mozdefstats['details'] = dict(counts=list())
for bucket in results.aggs['category']['buckets']:
entry = dict()
entry[bucket['key']] = bucket['doc_count']
mozdefstats['details']['counts'].append(entry)
return mozdefstats
except pyes.exceptions.NoServerAvailable:
logger.error(
'Elastic Search server could not be reached, check network connectivity'
)
def esSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.aggregationminutes))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
# search for aggregated event stats summaries within the date range
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q = pyes.FilteredQuery(q,pyes.BoolFilter(must=[qDate,
pyes.TermFilter('_type', 'mozdefstats')]))
results=es.search(query=q,size=100,indices=['events','events-previous'])
#avoid pyes iteration bug
rawresults = results._search_raw()
#examine the results
#for each details.counts, append the count
#as a list to the stats dict
stats=dict()
for r in rawresults['hits']['hits']:
for i in r['_source']['details']['counts']:
#print(i.values()[0])
if i.keys()[0] not in stats.keys():
stats[i.keys()[0]]=list()
stats[i.keys()[0]].append(i.values()[0])
# make a dictionairy of user-defined
# aggregation threshold percentages
aggregationthresholds = dict(zip(options.aggregations,
options.aggregationthresholds))
#for our running history of counts per category
#do some simple stats math to see if we
#should alert on anything
for s in stats:
alert = False
smean=round(numpy.mean(stats[s]))
sstd=round(numpy.std(stats[s]))
stat = round((sstd/smean)*100, 2)
if s in aggregationthresholds.keys():
if stat > aggregationthresholds[s]:
alert = True
elif stat > options.defaultthreshold:
alert = True
if alert:
print('{0} {1}%: \n\t\t{2} \n\t\t{3} \n\t\t{4}'.format(
s,
stat,
stats[s],
smean,
sstd
)
)
except pyes.exceptions.NoServerAvailable:
logger.error('Elastic Search server could not be reached, check network connectivity')
def esLdapResults(begindateUTC=None, enddateUTC=None):
'''an ES query/facet to count success/failed logins'''
resultsList = list()
if begindateUTC is None:
begindateUTC = datetime.now() - timedelta(hours=1)
begindateUTC = toUTC(begindateUTC)
if enddateUTC is None:
enddateUTC = datetime.now()
enddateUTC = toUTC(enddateUTC)
try:
es = pyes.ES((list('{0}'.format(s) for s in options.esservers)))
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',
from_value=begindateUTC, to_value=enddateUTC))
q = pyes.MatchAllQuery()
q = pyes.FilteredQuery(q, qDate)
q = pyes.FilteredQuery(q, pyes.TermFilter('tags', 'ldap'))
q = pyes.FilteredQuery(q,
pyes.TermFilter('details.result', 'ldap_invalid_credentials'))
q2 = q.search()
q2.facet.add_term_facet('details.result')
q2.facet.add_term_facet('details.dn', size=20)
results = es.search(q2, indices='events')
stoplist = ('o', 'mozilla', 'dc', 'com', 'mozilla.com',
'mozillafoundation.org', 'org')
for t in results.facets['details.dn'].terms:
if t['term'] in stoplist:
continue
#print(t['term'])
failures = 0
success = 0
dn = t['term']
#re-query with the terms of the details.dn
qt = pyes.MatchAllQuery()
qt = pyes.FilteredQuery(qt, qDate)
qt = pyes.FilteredQuery(qt, pyes.TermFilter('tags', 'ldap'))
qt = pyes.FilteredQuery(qt,
pyes.TermFilter('details.dn', t['term']))
qt2 = qt.search()
qt2.facet.add_term_facet('details.result')
results = es.search(qt2)
#sys.stdout.write('{0}\n'.format(results.facets['details.result'].terms))
for t in results.facets['details.result'].terms:
#print(t['term'],t['count'])
if t['term'] == 'ldap_success':
success = t['count']
if t['term'] == 'ldap_invalid_credentials':
failures = t['count']
resultsList.append(dict(dn=dn, failures=failures,
success=success, begin=begindateUTC.isoformat(),
end=enddateUTC.isoformat()))
return(json.dumps(resultsList))
except pyes.exceptions.NoServerAvailable:
sys.stderr.write('Elastic Search server could not be reached, check network connectivity\n')
def execute(self):
conn = pyes.ES(self.elastic_hosts)
queries = []
filters = []
filters.append(pyes.TermFilter('account', self.account))
for field, val in self.conditions:
if val != '':
queries.append(pyes.TextQuery(field, val))
if self.type not in [None, '']:
queries.append(pyes.TextQuery('type', self.type))
if self.path not in [None, '']:
if self.recursive:
filters.append(pyes.PrefixFilter('path', '%s/' % self.path))
else:
queries.append(pyes.TermQuery('dir', self.path))
if self.marker not in [None, '']:
filters.append(
pyes.RangeFilter(pyes.ESRangeOp('name', 'gt', self.marker)))
q = pyes.MatchAllQuery()
if len(queries) > 0:
q = pyes.BoolQuery(queries)
q = pyes.FilteredQuery(q, pyes.ANDFilter(filters))
self.logger.info("Running query: %s" % q.serialize())
results = conn.search(q, self.search_index_name, start=self.start,
size=self.limit)
if self.sort not in [None, '']:
sort_list = []
for row in self.sort.split(','):
sort_data = row.split(' ')
prefix = ""
if len(sort_data) > 1 and sort_data[1].lower() == 'desc':
prefix = "-"
if sort_data[0] in SORT_WHITELIST:
sort_list.append("{0}{1}".format(prefix, sort_data[0]))
if sort_list:
results.order_by(sort_list)
return results
def search_err(self, program, receiver):
start = "%s CST" % (datetime.astimezonenow() - timedelta(
hours=self.config['RANGE'])).daysstrftime('%Y/%m/%d %H:%M:%S')
end = "%s CST" % datetime.astimezonenow().strftime('%Y/%m/%d %H:%M:%S')
index = "%s_%s" % (program, datetime.astimezonenow().strftime('%Y-%m'))
q = pyes.RangeQuery(
pyes.ESRange('datetime', from_value=start, to_value=end))
q = pyes.FilteredQuery(q, pyes.TermQuery("level", "EROR"))
conn = pyes.ES(self.config['ES'])
results = conn.search(indices=index, query=q)
num = len(results)
if num == 0:
return
_content = ""
for r in results:
_content += "%s, %s<br/>" % (r['message'], r['datetime'])
title = self.config['MAIL']['MAIL_TITLE'] % program
content = "<b>Program:%s</b></b><br/>From:%s<br/>To:%s<br/><b>Total Num: %d</b><br/><br/><b>Content are:</b><br/>%s" % (
program, start, end, num, _content)
self.sendmail(receiver, title, content)