def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'event'),
pyes.TermFilter('eventsource', 'systemslogs'),
pyes.ExistsFilter('details.sourceipaddress'),
pyes.QueryFilter(pyes.MatchQuery('summary', 'failed', 'phrase')),
pyes.TermFilter('program', 'sshd'),
pyes.QueryFilter(
pyes.MatchQuery('summary', 'login ldap_count_entries',
'boolean'))
]
must_not = [
pyes.QueryFilter(
pyes.MatchQuery('summary', '10.22.8.128', 'phrase')),
pyes.QueryFilter(pyes.MatchQuery('summary', '10.8.75.35',
'phrase')),
pyes.QueryFilter(
pyes.MatchQuery('summary', '208.118.237.', 'phrase'))
]
self.filtersManual(date_timedelta, must=must, must_not=must_not)
# Search aggregations on field 'sourceipaddress', keep 50 samples of events at most
self.searchEventsAggreg('sourceipaddress', samplesLimit=50)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=10)
def searchESForBROAttackers(es, threshold):
begindateUTC = toUTC(datetime.now() - timedelta(hours=2))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
qBro = pyes.QueryFilter(pyes.MatchQuery('category', 'bro_notice',
'phrase'))
qErr = pyes.QueryFilter(
pyes.MatchQuery('details.note',
'MozillaHTTPErrors::Excessive_HTTP_Errors_Attacker',
'phrase'))
q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qBro, qErr]))
results = es.search(q, size=1000, indices='events,events-previous')
# grab results as native es results to avoid pyes iteration bug
# and get easier access to es metadata fields
rawresults = results._search_raw()['hits']['hits']
# Hit count is buried in the 'sub' field
# as: 'sub': u'6 in 1.0 hr, eps: 0'
# cull the records for hitcounts over the threshold before returning
attackers = list()
for r in rawresults:
hitcount = int(r['_source']['details']['sub'].split()[0])
if hitcount > threshold:
attackers.append(r)
return attackers
def main(self):
date_timedelta = dict(minutes=30)
self.config_file = './unauth_ssh_pyes.conf'
self.config = None
self.initConfiguration()
must = [
pyes.TermFilter('_type', 'event'),
pyes.TermFilter('category', 'syslog'),
pyes.TermFilter('details.program', 'sshd'),
pyes.QueryFilter(
pyes.QueryStringQuery('details.hostname: /{}/'.format(
self.config.hostfilter))),
pyes.QueryFilter(
pyes.MatchQuery('summary',
'Accepted publickey {}'.format(
self.config.user),
operator='and'))
]
must_not = []
for x in self.config.skiphosts:
must_not.append(pyes.QueryFilter(pyes.MatchQuery('summary', x)))
self.filtersManual(date_timedelta, must=must, must_not=must_not)
self.searchEventsSimple()
self.walkEvents()
def search(query, page):
title_q=pyes.MatchQuery('title', query)
question_q=pyes.MatchQuery('question', query)
title_results=kunjika.es_conn.search(query=title_q)
question_results=kunjika.es_conn.search(query=question_q)
results=[]
for r in title_results:
results.append(r['qid'])
for r in question_results:
results.append(r['qid'])
return common_rendering(results, query, page)
def get_autocomplete(request):
#print request.args.get('val')
q=pyes.MatchQuery('title', request.args.get('val'))
title_results=kunjika.es_conn.search(query=q)
results=[]
for r in title_results:
results.append(r['qid'])
questions_list = []
if len(results) > 10:
for i in range(0, len(results)):
question = kunjika.qb.get(str(results[i])).value
questions_list.append({'title':question['title'], 'qid': question['qid'], 'url':question['content']['url']})
#print question['title'] + ' ' + question['content']['url'] + str(question['qid'])
else:
for i in range(0, len(results)):
question = kunjika.qb.get(str(results[i])).value
questions_list.append({'title':question['title'], 'qid': question['qid'], 'url':question['content']['url']})
#print question['title'] + ' ' + question['content']['url'] + str(question['qid'])
if len(results) > 10:
return jsonify({'data': questions_list[0:10]})
else:
return jsonify({'data':questions_list[0:len(results)]})
def search_title(query, page):
title=query[6:]
q=pyes.MatchQuery('title', title)
title_results=kunjika.es_conn.search(query=q)
results=[]
for r in title_results:
results.append(r['qid'])
#print str(r)
return common_rendering(results, query, page)
def main(self):
# look for events in last X mins
date_timedelta = dict(minutes=2)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'event'),
pyes.QueryFilter(pyes.MatchQuery('summary','failed','phrase')),
pyes.TermFilter('program','sshd'),
pyes.QueryFilter(pyes.MatchQuery('summary', 'login invalid ldap_count_entries', 'boolean'))
]
must_not = [
pyes.QueryFilter(pyes.MatchQuery('summary','10.22.75.203','phrase')),
pyes.QueryFilter(pyes.MatchQuery('summary','10.8.75.144','phrase'))
]
self.filtersManual(date_timedelta, must=must, must_not=must_not)
# Search aggregations on field 'sourceipaddress', keep X samples of events at most
self.searchEventsAggreg('sourceipaddress', samplesLimit=10)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=10)
def search_description(query, page):
description=query[12:]
q=pyes.MatchQuery('description', description)
question_results=kunjika.es_conn.search(query=q)
results=[]
for r in question_results:
results.append(r['qid'])
return common_rendering(results, query, page)
def main(self):
# look for events in last 10 mins
date_timedelta = dict(minutes=10)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'event'),
pyes.TermFilter('program', 'fail2ban'),
pyes.QueryFilter(pyes.MatchQuery("summary","has been banned","phrase"))
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.QueryFilter(pyes.MatchQuery('summary','Failsafe Duo login','phrase'))
]
self.filtersManual(date_timedelta, must=must)
# Search aggregations on field 'sourceipaddress', keep X samples of events at most
self.searchEventsAggregated('details.hostname', samplesLimit=10)
# alert when >= X matching events in an aggregation
# in this case, always
self.walkAggregations(threshold=1)
def main(self):
# look for events in last x
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('category', 'ldapChange'),
pyes.TermFilter('changetype', 'modify'),
pyes.QueryFilter(pyes.MatchQuery("summary","groups"))
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
# look for events in last x
date_timedelta = dict(minutes=15)
# Configure filters
# looking for pwdAccountLockedTime setting by admin
must = [
pyes.TermFilter('category', 'ldapChange'),
pyes.TermFilter("actor", "cn=admin,dc=mozilla"),
pyes.QueryFilter(pyes.MatchQuery('changepairs', 'replace:pwdAccountLockedTime','phrase'))
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def search_user(query, page):
(qcount, acount, tcount, ucount, tag_list) = common_data()
user=query[5:]
q=pyes.MatchQuery('name', user)
question_results=kunjika.es_conn.search(query=q)
results=[]
for r in question_results:
print r
results.append(r['uid'])
questions_list=[]
for uid in results:
question_view = urllib2.urlopen(
kunjika.DB_URL + 'questions/_design/dev_qa/_view/get_questions_by_userid?key=' + '"' +str(uid) + '"').read()
question_view = json.loads(question_view)['rows']
print question_view
for row in question_view:
print row['value']
question = {}
question['qid'] = row['value'][0]
question['votes'] = row['value'][1]
question['acount'] = row['value'][2]
question['title'] = row['value'][3]
question['url'] = row['value'][4]
question['views'] = row['value'][5]
question['ts'] = row['value'][6]
question['op'] = row['value'][7]
questions_list.append(question)
for i in questions_list:
i['tstamp'] = strftime("%a, %d %b %Y %H:%M", localtime(i['ts']))
user = kunjika.cb.get(i['op']).value
i['opname'] = user['name']
pagination = Pagination(page, kunjika.QUESTIONS_PER_PAGE, len(questions_list))
if g.user is None:
return render_template('search.html', title='Search results for ' + query, qpage=True, questions=questions_list[(page-1)*kunjika.QUESTIONS_PER_PAGE:],
pagination=pagination, qcount=qcount, ucount=ucount, tcount=tcount, acount=acount, tag_list=tag_list, query=query)
elif g.user is not None and g.user.is_authenticated():
return render_template('search.html', title='Search results for ' + query, qpage=True, questions=questions_list[(page-1)*kunjika.QUESTIONS_PER_PAGE:],
pagination=pagination, qcount=qcount, ucount=ucount, tcount=tcount, acount=acount, tag_list=tag_list, query=query)
else:
return render_template('search.html', title='Search results for ' + query, qpage=True, questions=questions_list[(page-1)*kunjika.QUESTIONS_PER_PAGE:],
pagination=pagination, qcount=qcount, ucount=ucount, tcount=tcount, acount=acount, tag_list=tag_list, query=query)
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('tags', 'nubis_events_non_prod'),
pyes.TermFilter('tags', 'nubis_events_prod'),
pyes.TermFilter('category', 'syslog'),
pyes.TermFilter('details.__tag', 'ec2.forward.squid.access'),
pyes.QueryFilter(pyes.MatchQuery('details.summary','is DENIED, because it matched','phrase')),
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def get_similar_questions(title, qid):
print title
title_q=pyes.MatchQuery('title', title)
title_results=kunjika.es_conn.search(query=title_q)
results=[]
for r in title_results:
print r
if r['qid'] != qid:
question_dict = {}
question_dict = question.get_question_by_id(str(r['qid']), question_dict)
print question_dict
results.append([r['qid'], r['title'], question_dict['content']['url']])
return results[:10]
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'bro'),
pyes.TermFilter('eventsource', 'nsm'),
pyes.TermFilter('category', 'bronotice'),
pyes.ExistsFilter('details.sourceipaddress'),
pyes.QueryFilter(pyes.MatchQuery('details.note','SSH::Password_Guessing','phrase')),
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
# look for events in last X mins
date_timedelta = dict(minutes=2)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'bro'),
pyes.TermFilter('eventsource', 'nsm'),
pyes.TermFilter('category', 'brointel'),
pyes.ExistsFilter('seenindicator'),
pyes.QueryFilter(pyes.MatchQuery('hostname', 'sensor1 sensor2 sensor3', 'boolean'))
]
self.filtersManual(date_timedelta, must=must)
# Search aggregations on field 'seenindicator', keep X samples of events at most
self.searchEventsAggregated('details.seenindicator', samplesLimit=10)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=10)
def esSearch(es, macassignments=None, begindateUTC=None, enddateUTC=None):
'''
Search ES for an event that ties a username to a mac address
This example searches for junos wifi correlations on authentication success
Expecting an event like: user: [email protected]; mac: 5c:f9:38:b1:de:cf; author reason: roamed session; ssid: ANSSID; AP 46/2\n
'''
usermacre=re.compile(r'''user: (?P<username>.*?); mac: (?P<macaddress>.*?); ''',re.IGNORECASE)
correlations={} # list of dicts to populate hits we find
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.correlationminutes))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
# search for events within the date range
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q=pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(pyes.BoolFilter(must=[
qDate,
pyes.TermFilter("program","AUTHORIZATION-SUCCESS")
],
must_not=[
pyes.QueryFilter(
pyes.MatchQuery("summary","last-resort","phrase")
)
]))
results = es.search(q, size=10000, indices=['events', 'events-previous'])
rawresults=results._search_raw()
for r in rawresults['hits']['hits']:
fields = re.search(usermacre,r['_source']['summary'])
if fields:
if '{0} {1}'.format(fields.group('username'),fields.group('macaddress')) not in correlations.keys():
if fields.group('macaddress')[0:8].lower() in macassignments.keys():
entity=macassignments[fields.group('macaddress')[0:8].lower()]
else:
entity='unknown'
correlations['{0} {1}'.format(fields.group('username'),fields.group('macaddress'))]=dict(username=fields.group('username'),
macaddress=fields.group('macaddress'),
entity=entity,
utctimestamp=r['_source']['utctimestamp'])
return correlations
except pyes.exceptions.NoServerAvailable:
logger.error('Elastic Search server could not be reached, check network connectivity')
def search_tag(query, page):
(qcount, acount, tcount, ucount, tag_list) = common_data()
tag=query[4:]
q=pyes.MatchQuery('tag', tag)
question_results=kunjika.es_conn.search(query=q)
results=[]
for r in question_results:
results.append(r['tag'])
print r['tag']
questions_list=[]
for tag in results:
#question_view = urllib2.urlopen(
# kunjika.DB_URL + 'questions/_design/dev_qa/_view/get_questions_by_tag?key=' + '"' + tag + '"').read()
#question_view = json.loads(question_view)['rows']
#print question_view
q = Query(key=tag)
for result in View(kunjika.qb, "dev_qa", "get_questions_by_tag", include_docs=True, query=q):
#print result
questions_list.append(result.doc.value)
#for element in question_view['rows']:
# questions_list.append(element['value'])
for i in questions_list:
i['tstamp'] = strftime("%a, %d %b %Y %H:%M", localtime(i['content']['ts']))
user = kunjika.cb.get(i['content']['op']).value
i['opname'] = user['name']
pagination = Pagination(page, kunjika.QUESTIONS_PER_PAGE, len(questions_list))
if g.user is None:
return render_template('search.html', title='Search results for ' + query, qpage=True, questions=questions_list[(page-1)*kunjika.QUESTIONS_PER_PAGE:],
pagination=pagination, qcount=qcount, ucount=ucount, tcount=tcount, acount=acount, tag_list=tag_list, query=query)
elif g.user is not None and g.user.is_authenticated():
return render_template('search.html', title='Search results for ' + query, qpage=True, questions=questions_list[(page-1)*kunjika.QUESTIONS_PER_PAGE:],
pagination=pagination, qcount=qcount, ucount=ucount, tcount=tcount, acount=acount, tag_list=tag_list, query=query)
else:
return render_template('search.html', title='Search results for ' + query, qpage=True, questions=questions_list[(page-1)*kunjika.QUESTIONS_PER_PAGE:],
pagination=pagination, qcount=qcount, ucount=ucount, tcount=tcount, acount=acount, tag_list=tag_list, query=query)
def searchForSSHKeys(es):
begindateUTC = toUTC(datetime.now() - timedelta(minutes=5))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermFilter("program", "sshd")
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(must=[
qType, qDate, qEvents,
pyes.QueryFilter(
pyes.MatchQuery("summary",
"found matching key accepted publickey",
"boolean"))
]))
results = es.search(q, size=10000, indices='events')
# return raw search to avoid pyes iteration bug
return results._search_raw()
def filtersFromKibanaDash(self, fp, date_timedelta):
"""
Import filters from a kibana dashboard
fp is the file path of the json file
date_timedelta is a dict in timedelta format
see https://docs.python.org/2/library/datetime.html#timedelta-objects
"""
f = open(fp)
data = json.load(f)
must = []
should = []
must_not = []
for filtid in data['services']['filter']['list'].keys():
filt = data['services']['filter']['list'][filtid]
if filt['active'] and 'query' in filt.keys():
value = filt['query'].split(':')[-1]
fieldname = filt['query'].split(':')[0].split('.')[-1]
# self.log.info(fieldname)
# self.log.info(value)
# field: fieldname
# query: value
if 'field' in filt.keys():
fieldname = filt['field']
value = filt['query']
if '\"' in value:
value = value.split('\"')[1]
pyesfilt = pyes.QueryFilter(
pyes.MatchQuery(fieldname, value, 'phrase'))
else:
pyesfilt = pyes.TermFilter(fieldname, value)
else:
# _exists_:field
if filt['query'].startswith('_exists_:'):
pyesfilt = pyes.ExistsFilter(value.split('.')[-1])
# self.log.info('exists %s' % value.split('.')[-1])
# _missing_:field
elif filt['query'].startswith('_missing_:'):
pyesfilt = pyes.filters.MissingFilter(
value.split('.')[-1])
# self.log.info('missing %s' % value.split('.')[-1])
# field:"value"
elif '\"' in value:
value = value.split('\"')[1]
pyesfilt = pyes.QueryFilter(
pyes.MatchQuery(fieldname, value, 'phrase'))
# self.log.info("phrase %s %s" % (fieldname, value))
# field:(value1 value2 value3)
elif '(' in value and ')' in value:
value = value.split('(')[1]
value = value.split('(')[0]
pyesfilt = pyes.QueryFilter(
pyes.MatchQuery(fieldname, value, "boolean"))
# field:value
else:
pyesfilt = pyes.TermFilter(fieldname, value)
# self.log.info("terms %s %s" % (fieldname, value))
if filt['mandate'] == 'must':
must.append(pyesfilt)
elif filt['mandate'] == 'either':
should.append(pyesfilt)
elif filt['mandate'] == 'mustNot':
must_not.append(pyesfilt)
# self.log.info(must)
f.close()
self.filtersManual(date_timedelta,
must=must,
should=should,
must_not=must_not)
def march_query_tag(field, query, sub_type):
must = pyes.TermQuery("sub_type", sub_type)
should = pyes.MatchQuery(field, query)
query = pyes.BoolQuery(must=must, should=should)
return [i for i in CONN_ES.search(query=query)]
