def esLdapResults(begindateUTC=None, enddateUTC=None):
'''an ES query/facet to count success/failed logins'''
resultsList = list()
if begindateUTC is None:
begindateUTC = datetime.now() - timedelta(hours=1)
begindateUTC = toUTC(begindateUTC)
if enddateUTC is None:
enddateUTC = datetime.now()
enddateUTC = toUTC(enddateUTC)
try:
es = pyes.ES((list('{0}'.format(s) for s in options.esservers)))
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',
from_value=begindateUTC, to_value=enddateUTC))
q = pyes.MatchAllQuery()
q = pyes.FilteredQuery(q, qDate)
q = pyes.FilteredQuery(q, pyes.TermFilter('tags', 'ldap'))
q = pyes.FilteredQuery(q,
pyes.TermFilter('details.result', 'ldap_invalid_credentials'))
q2 = q.search()
q2.facet.add_term_facet('details.result')
q2.facet.add_term_facet('details.dn', size=20)
results = es.search(q2, indices='events')
stoplist = ('o', 'mozilla', 'dc', 'com', 'mozilla.com',
'mozillafoundation.org', 'org')
for t in results.facets['details.dn'].terms:
if t['term'] in stoplist:
continue
#print(t['term'])
failures = 0
success = 0
dn = t['term']
#re-query with the terms of the details.dn
qt = pyes.MatchAllQuery()
qt = pyes.FilteredQuery(qt, qDate)
qt = pyes.FilteredQuery(qt, pyes.TermFilter('tags', 'ldap'))
qt = pyes.FilteredQuery(qt,
pyes.TermFilter('details.dn', t['term']))
qt2 = qt.search()
qt2.facet.add_term_facet('details.result')
results = es.search(qt2)
#sys.stdout.write('{0}\n'.format(results.facets['details.result'].terms))
for t in results.facets['details.result'].terms:
#print(t['term'],t['count'])
if t['term'] == 'ldap_success':
success = t['count']
if t['term'] == 'ldap_invalid_credentials':
failures = t['count']
resultsList.append(dict(dn=dn, failures=failures,
success=success, begin=begindateUTC.isoformat(),
end=enddateUTC.isoformat()))
return(json.dumps(resultsList))
except pyes.exceptions.NoServerAvailable:
sys.stderr.write('Elastic Search server could not be reached, check network connectivity\n')
def esSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() -
timedelta(minutes=options.aggregationminutes))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
# search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qDate]))
q = q.search()
qagg = pyes.aggs.TermsAgg(name='category', field='category')
q.agg.add(qagg)
results = es.search(query=q, indices=['events'])
mozdefstats = dict(utctimestamp=toUTC(datetime.now()).isoformat())
mozdefstats['summary'] = 'Aggregated category counts'
mozdefstats['processid'] = os.getpid()
mozdefstats['processname'] = sys.argv[0]
mozdefstats['details'] = dict(counts=list())
for bucket in results.aggs['category']['buckets']:
entry = dict()
entry[bucket['key']] = bucket['doc_count']
mozdefstats['details']['counts'].append(entry)
return mozdefstats
except pyes.exceptions.NoServerAvailable:
logger.error(
'Elastic Search server could not be reached, check network connectivity'
)
def get_object_list(self, request):
offset = long(request.GET.get("offset", 0))
limit = long(request.GET.get("limit", self._meta.limit))
sort = self.get_sorting(request)
q = request.GET.get("q")
if q:
query = pyes.StringQuery(q)
else:
query = pyes.MatchAllQuery()
size = (limit + offset) - (1 if offset else 0)
start = offset + (2 if offset >= limit else 1)
print "offset ", offset, limit.__class__
print "limit ", limit, limit.__class__
print "start ", start, start.__class__
print "size ", size, size.__class__
print "sort", sort
# refresh the index before query
self.es.refresh(self._meta.indices[0])
search = pyes.query.Search(query=query,
start=start,
size=size,
sort=sort)
results = self.es.search(search, indices=self._meta.indices)
return results
def searchESForBROAttackers(es, threshold):
begindateUTC = toUTC(datetime.now() - timedelta(hours=2))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
qBro = pyes.QueryFilter(pyes.MatchQuery('category', 'bro_notice',
'phrase'))
qErr = pyes.QueryFilter(
pyes.MatchQuery('details.note',
'MozillaHTTPErrors::Excessive_HTTP_Errors_Attacker',
'phrase'))
q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qBro, qErr]))
results = es.search(q, size=1000, indices='events,events-previous')
# grab results as native es results to avoid pyes iteration bug
# and get easier access to es metadata fields
rawresults = results._search_raw()['hits']['hits']
# Hit count is buried in the 'sub' field
# as: 'sub': u'6 in 1.0 hr, eps: 0'
# cull the records for hitcounts over the threshold before returning
attackers = list()
for r in rawresults:
hitcount = int(r['_source']['details']['sub'].split()[0])
if hitcount > threshold:
attackers.append(r)
return attackers
def esSearch(es, begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.aggregationminutes))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
# search for aggregated event stats summaries within the date range
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q = pyes.FilteredQuery(q,pyes.BoolFilter(must=[qDate,
pyes.TermFilter('_type', 'mozdefstats')]))
results=es.search(query=q,size=100,indices=['events','events-previous'])
#avoid pyes iteration bug
rawresults = results._search_raw()
#examine the results
#for each details.counts, append the count
#as a list to the stats dict
stats=dict()
for r in rawresults['hits']['hits']:
for i in r['_source']['details']['counts']:
#print(i.values()[0])
if i.keys()[0] not in stats.keys():
stats[i.keys()[0]]=list()
stats[i.keys()[0]].append(i.values()[0])
# make a dictionairy of user-defined
# aggregation threshold percentages
aggregationthresholds = dict(zip(options.aggregations,
options.aggregationthresholds))
#for our running history of counts per category
#do some simple stats math to see if we
#should alert on anything
for s in stats:
alert = False
smean=round(numpy.mean(stats[s]))
sstd=round(numpy.std(stats[s]))
stat = round((sstd/smean)*100, 2)
if s in aggregationthresholds.keys():
if stat > aggregationthresholds[s]:
alert = True
elif stat > options.defaultthreshold:
alert = True
if alert:
print('{0} {1}%: \n\t\t{2} \n\t\t{3} \n\t\t{4}'.format(
s,
stat,
stats[s],
smean,
sstd
)
)
except pyes.exceptions.NoServerAvailable:
logger.error('Elastic Search server could not be reached, check network connectivity')
def naslagview(request):
es = pyes.ES(settings.ELASTIC_SEARCH)
r = es.search(pyes.MatchAllQuery(),
indices='nedind',
doc_types=['naslag'],
sort="boekcode")
return direct_to_template(request, "naslag.html", {'naslag': r})
def filtersManual(self, date_timedelta, must=[], should=[], must_not=[]):
"""
Configure filters manually
date_timedelta is a dict in timedelta format
see https://docs.python.org/2/library/datetime.html#timedelta-objects
must, should and must_not are pyes filter objects lists
see http://pyes.readthedocs.org/en/latest/references/pyes.filters.html
"""
self.begindateUTC = toUTC(datetime.now() - timedelta(**date_timedelta))
self.enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',
from_value=self.begindateUTC, to_value=self.enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
#Don't fire on already alerted events
if pyes.ExistsFilter('alerttimestamp') not in must_not:
must_not.append(pyes.ExistsFilter('alerttimestamp'))
must.append(qDate)
q.filters.append(pyes.BoolFilter(
must=must,
should=should,
must_not=must_not))
self.filter = q
def load(self):
"""load all json documents from ES:toolbox/projects"""
query = pyes.MatchAllQuery()
results = self.es_query(query)
for hit in results['hits']['hits']:
self.update(hit['_source'], True)
def test_delete_warmer(self):
warmer1 = pyes.Search(pyes.MatchAllQuery())
self.conn.put_warmer(indices=[self.index_name],
name='w1',
warmer=warmer1)
self.conn.delete_warmer(indices=[self.index_name], name='w1')
self.assertRaises(pyes.exceptions.ElasticSearchException,
self.conn.get_warmer,
indices=[self.index_name],
name='w1')
def getESAlerts(es):
begindateUTC = toUTC(datetime.now() - timedelta(minutes=50))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'alert')
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(pyes.BoolFilter(must=[qDate, qType]))
results = es.search(q, size=10000, indices='alerts')
# return raw search to avoid pyes iteration bug
return results._search_raw()
def query(self,
sort='timestamp',
start=0,
size=20,
severity=None,
timestamp_from=None,
timestamp_till=None):
fltr = []
if severity is not None:
fltr.append(
pyes.TermFilter(field='severity', value=severity)
)
if timestamp_from is not None:
if isinstance(timestamp_from, datetime.datetime):
timestamp_from = timestamp_from.isoformat()
fltr.append(
pyes.RangeFilter(
pyes.ESRangeOp(
'timestamp', 'gte', timestamp_from
)
)
)
if timestamp_till is not None:
if isinstance(timestamp_till, datetime.datetime):
timestamp_till = timestamp_till.isoformat()
fltr.append(
pyes.RangeFilter(
pyes.ESRangeOp(
'timestamp', 'lte', timestamp_till
)
)
)
f = None
if fltr:
f = pyes.ANDFilter(fltr)
q = pyes.MatchAllQuery()
s = pyes.Search(
query=q,
filter=f,
start=start,
size=size)
return self.es.search(
s,
indices=[self.index],
doc_types=[self.document_type])
def esBroXSSEvents():
begindateUTC = toUTC(datetime.now() - timedelta(minutes=30))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermFilter("category", "broxsslog")
qalerted = pyes.ExistsFilter('alerttimestamp')
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(must=[qType, qDate, qEvents,
pyes.ExistsFilter('uri')],
must_not=[qalerted]))
return q
def esBroIntelEvents():
begindateUTC = toUTC(datetime.now() - timedelta(minutes=30))
enddateUTC = toUTC(datetime.now())
#search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermsFilter('category', ['brointel'])
qalerted = pyes.ExistsFilter('alerttimestamp')
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(
must=[qType, qDate, qEvents,
pyes.ExistsFilter('seenindicator')],
must_not=[qalerted]))
return q
def esSearch(es, macassignments=None, begindateUTC=None, enddateUTC=None):
'''
Search ES for an event that ties a username to a mac address
This example searches for junos wifi correlations on authentication success
Expecting an event like: user: [email protected]; mac: 5c:f9:38:b1:de:cf; author reason: roamed session; ssid: ANSSID; AP 46/2\n
'''
usermacre=re.compile(r'''user: (?P<username>.*?); mac: (?P<macaddress>.*?); ''',re.IGNORECASE)
correlations={} # list of dicts to populate hits we find
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.correlationminutes))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
# search for events within the date range
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q=pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(pyes.BoolFilter(must=[
qDate,
pyes.TermFilter("program","AUTHORIZATION-SUCCESS")
],
must_not=[
pyes.QueryFilter(
pyes.MatchQuery("summary","last-resort","phrase")
)
]))
results = es.search(q, size=10000, indices=['events', 'events-previous'])
rawresults=results._search_raw()
for r in rawresults['hits']['hits']:
fields = re.search(usermacre,r['_source']['summary'])
if fields:
if '{0} {1}'.format(fields.group('username'),fields.group('macaddress')) not in correlations.keys():
if fields.group('macaddress')[0:8].lower() in macassignments.keys():
entity=macassignments[fields.group('macaddress')[0:8].lower()]
else:
entity='unknown'
correlations['{0} {1}'.format(fields.group('username'),fields.group('macaddress'))]=dict(username=fields.group('username'),
macaddress=fields.group('macaddress'),
entity=entity,
utctimestamp=r['_source']['utctimestamp'])
return correlations
except pyes.exceptions.NoServerAvailable:
logger.error('Elastic Search server could not be reached, check network connectivity')
def execute(self):
conn = pyes.ES(self.elastic_hosts)
queries = []
filters = []
filters.append(pyes.TermFilter('account', self.account))
for field, val in self.conditions:
if val != '':
queries.append(pyes.TextQuery(field, val))
if self.type not in [None, '']:
queries.append(pyes.TextQuery('type', self.type))
if self.path not in [None, '']:
if self.recursive:
filters.append(pyes.PrefixFilter('path', '%s/' % self.path))
else:
queries.append(pyes.TermQuery('dir', self.path))
if self.marker not in [None, '']:
filters.append(
pyes.RangeFilter(pyes.ESRangeOp('name', 'gt', self.marker)))
q = pyes.MatchAllQuery()
if len(queries) > 0:
q = pyes.BoolQuery(queries)
q = pyes.FilteredQuery(q, pyes.ANDFilter(filters))
self.logger.info("Running query: %s" % q.serialize())
results = conn.search(q, self.search_index_name, start=self.start,
size=self.limit)
if self.sort not in [None, '']:
sort_list = []
for row in self.sort.split(','):
sort_data = row.split(' ')
prefix = ""
if len(sort_data) > 1 and sort_data[1].lower() == 'desc':
prefix = "-"
if sort_data[0] in SORT_WHITELIST:
sort_list.append("{0}{1}".format(prefix, sort_data[0]))
if sort_list:
results.order_by(sort_list)
return results
def kibanaDashboards():
try:
resultsList = []
es = pyes.ES((list('{0}'.format(s) for s in options.esservers)))
r = es.search(pyes.Search(pyes.MatchAllQuery(), size=100),
'kibana-int', 'dashboard')
if r:
for dashboard in r:
dashboardJson = json.loads(dashboard.dashboard)
resultsList.append({
'name': dashboardJson['title'],
'url': "%s/%s/%s" % (options.kibanaurl,
"index.html#/dashboard/elasticsearch",
dashboardJson['title'])
})
return json.dumps(resultsList)
else:
sys.stderr.write('No Kibana dashboard found\n')
except pyes.exceptions.NoServerAvailable:
sys.stderr.write('Elastic Search server could not be reached, check network connectivity\n')
def get_search_results(q, extra_filter=None, sort='trefwoord'):
es = pyes.ES(settings.ELASTIC_SEARCH)
if q:
q1 = pyes.StringQuery(q)
else:
q1 = pyes.MatchAllQuery()
if extra_filter:
q2 = q1.search(filter=extra_filter)
else:
q2 = q1.search()
q2.facet.add_term_facet('taal')
q2.facet.add_term_facet('woordsoort')
q2.facet.add_term_facet('sfeer', size="60")
results = es.search(query=q2,
indices='nedind',
doc_types=['trefwoord'],
sort=sort)
return results
def searchForSSHKeys(es):
begindateUTC = toUTC(datetime.now() - timedelta(minutes=5))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermFilter("program", "sshd")
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(must=[
qType, qDate, qEvents,
pyes.QueryFilter(
pyes.MatchQuery("summary",
"found matching key accepted publickey",
"boolean"))
]))
results = es.search(q, size=10000, indices='events')
# return raw search to avoid pyes iteration bug
return results._search_raw()
def test_put_get_warmer(self):
warmer1 = pyes.Search(pyes.MatchAllQuery())
#ES fails if the index is empty
self.conn.index({'a': 1}, self.index_name, self.document_type)
self.conn.refresh(self.index_name)
self.conn.put_warmer(indices=[self.index_name],
name='w1',
warmer=warmer1)
result = self.conn.get_warmer(indices=[self.index_name], name='w1')
expected = {
self.index_name: {
'warmers': {
'w1': {
'source': {
'query': {
'match_all': {}
}
},
'types': []
}
}
}
}
self.assertEqual(result, expected)
def list_display(request):
current_page_number = request.GET.get('page', 1)
form = forms.StorageSearchForm()
# get ElasticSearch stats
aip_indexed_file_count = advanced_search.indexed_count('aips')
# get AIPs
order_by = request.GET.get('order_by', 'name')
sort_by = request.GET.get('sort_by', 'up')
if sort_by == 'down':
sort_direction = 'desc'
else:
sort_direction = 'asc'
sort_specification = order_by + ':' + sort_direction
conn = elasticSearchFunctions.connect_and_create_index('aips')
items_per_page = 10
start = (int(current_page_number) - 1) * items_per_page
aipResults = conn.search(pyes.Search(pyes.MatchAllQuery(),
start=start,
size=items_per_page),
doc_types=['aip'],
fields='origin,uuid,filePath,created,name,size',
sort=sort_specification)
try:
len(aipResults)
except pyes.exceptions.ElasticSearchException:
# there will be an error if no mapping exists for AIPs due to no AIPs
# having been created
return render(request, 'archival_storage/archival_storage.html',
locals())
# handle pagination
page = helpers.pager(aipResults, items_per_page, current_page_number)
if not page:
raise Http404
# augment data
sips = []
for aip in page['objects']:
sip = {}
sip['href'] = aip.filePath.replace(AIPSTOREPATH + '/', "AIPsStore/")
sip['name'] = aip.name
sip['uuid'] = aip.uuid
sip['date'] = aip.created
try:
size = float(aip.size)
sip['size'] = '{0:.2f} MB'.format(size)
except:
sip['size'] = 'Removed'
sips.append(sip)
# get total size of all AIPS from ElasticSearch
q = pyes.MatchAllQuery().search()
q.facet.add(pyes.facets.StatisticalFacet('total', field='size'))
aipResults = conn.search(q, doc_types=['aip'])
total_size = aipResults.facets.total.total
total_size = '{0:.2f}'.format(total_size)
return render(request, 'archival_storage/archival_storage.html', locals())
#coding=utf-8
import pyes
import json
import FormatTranslator
import FileTools
'''------------------------------------------- Matrix to csv -------------------------------------------'''
conn = pyes.es.ES('localhost:9200')
q = pyes.MatchAllQuery()
tagg = pyes.aggs.TermsAgg('user_id', field='user_id', sub_aggs=[])
tagg1 = pyes.aggs.TermsAgg('name', field='name')
tagg.sub_aggs.append(tagg1)
qsearch = pyes.query.Search(q)
qsearch.agg.add(tagg)
rs = conn.search(query=qsearch, indices='example_index', type="example_type")
print json.dumps(rs.aggs, indent=2)
formatTranslator = FormatTranslator.FormatTranslator()
result = formatTranslator.ES_Aggs_2_Layer_to_Matrix_and_indice(
rs.aggs, "user_id", "name")
print result['rowIndexList']
print result['colIndexList']
print result['matrix']
fileTools = FileTools.FileTools()
fileTools.List_to_CSV(result['colIndexList'], "col_index.csv")
fileTools.Matrix_to_CSV(result['matrix'], "matrix.csv")
'''------------------------------------------- Agg to csv -------------------------------------------'''
