def esLdapResults(begindateUTC=None, enddateUTC=None):
'''an ES query/facet to count success/failed logins'''
resultsList = list()
if begindateUTC is None:
begindateUTC = datetime.now() - timedelta(hours=1)
begindateUTC = toUTC(begindateUTC)
if enddateUTC is None:
enddateUTC = datetime.now()
enddateUTC = toUTC(enddateUTC)
try:
es_client = ElasticsearchClient(list('{0}'.format(s) for s in options.esservers))
search_query = SearchQuery()
range_match = RangeMatch('utctimestamp', begindateUTC, enddateUTC)
search_query.add_must(range_match)
search_query.add_must(TermMatch('tags', 'ldap'))
search_query.add_must(TermMatch('details.result', 'LDAP_INVALID_CREDENTIALS'))
search_query.add_aggregation(Aggregation('details.result'))
search_query.add_aggregation(Aggregation('details.dn'))
results = search_query.execute(es_client, indices=['events'])
stoplist = ('o', 'mozilla', 'dc', 'com', 'mozilla.com', 'mozillafoundation.org', 'org', 'mozillafoundation')
for t in results['aggregations']['details.dn']['terms']:
if t['key'] in stoplist:
continue
failures = 0
success = 0
dn = t['key']
details_query = SearchQuery()
details_query.add_must(range_match)
details_query.add_must(TermMatch('tags', 'ldap'))
details_query.add_must(TermMatch('details.dn', dn))
details_query.add_aggregation(Aggregation('details.result'))
results = details_query.execute(es_client)
for t in results['aggregations']['details.result']['terms']:
if t['key'].upper() == 'LDAP_SUCCESS':
success = t['count']
if t['key'].upper() == 'LDAP_INVALID_CREDENTIALS':
failures = t['count']
resultsList.append(
dict(
dn=dn,
failures=failures,
success=success,
begin=begindateUTC.isoformat(),
end=enddateUTC.isoformat()
)
)
return(json.dumps(resultsList))
except Exception as e:
sys.stderr.write('Error trying to get ldap results: {0}\n'.format(e))
def test_multiple_aggregations(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('note'))
search_query.add_aggregation(Aggregation('test'))
results = search_query.execute(self.es_client)
aggregation_keys = results['aggregations'].keys()
aggregation_keys.sort()
assert aggregation_keys == ['note', 'test']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 2
assert results['aggregations']['note']['terms'][0].keys() == [
'count', 'key'
]
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0]['key'] == 'abvc'
assert results['aggregations']['note']['terms'][1]['count'] == 1
assert results['aggregations']['note']['terms'][1]['key'] == 'think'
assert results['aggregations']['test'].keys() == ['terms']
assert len(results['aggregations']['test']['terms']) == 1
assert results['aggregations']['test']['terms'][0].keys() == [
'count', 'key'
]
assert results['aggregations']['test']['terms'][0]['count'] == 4
assert results['aggregations']['test']['terms'][0]['key'] == 'value'
def test_aggregation_non_existing_term(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('example'))
results = search_query.execute(self.es_client)
assert sorted(results.keys()) == ['aggregations', 'hits', 'meta']
assert len(results['hits']) == 4
assert list(results['aggregations'].keys()) == ['example']
assert list(results['aggregations']['example'].keys()) == ['terms']
assert results['aggregations']['example']['terms'] == []
def test_aggregation_non_existing_layers_term(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('details.ipinformation'))
results = search_query.execute(self.es_client)
assert results['aggregations'].keys() == ['details.ipinformation']
assert results['aggregations']['details.ipinformation'].keys() == [
'terms'
]
assert len(
results['aggregations']['details.ipinformation']['terms']) == 0
def get_num_events(self):
self.refresh('events')
search_query = SearchQuery()
search_query.add_must(TermMatch('_type', 'event'))
search_query.add_aggregation(Aggregation('_type'))
results = search_query.execute(self.es_client)
if len(results['aggregations']['_type']['terms']) != 0:
return results['aggregations']['_type']['terms'][0]['count']
else:
return 0
def test_aggregation_with_aggregation_size(self):
for num in range(0, 100):
event = {'keyname': 'value' + str(num)}
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(ExistsMatch('keyname'))
search_query.add_aggregation(Aggregation('keyname', 2))
results = search_query.execute(self.es_client)
assert len(results['aggregations']['keyname']['terms']) == 2
def test_aggregation_without_must_fields(self):
event = self.generate_default_event()
event['_source']['utctimestamp'] = event['_source']['utctimestamp']()
event['_source']['receivedtimestamp'] = event['_source']['receivedtimestamp']()
self.populate_test_event(event)
self.refresh(self.event_index_name)
search_query = SearchQuery(minutes=10)
search_query.add_aggregation(Aggregation('source'))
results = search_query.execute(self.es_client)
assert results['aggregations']['source']['terms'][0]['count'] == 1
def test_aggregation_query_execute(self):
query = SearchQuery()
query.add_must(ExistsMatch('note'))
query.add_aggregation(Aggregation('note'))
assert query.date_timedelta == {}
self.populate_example_event()
self.populate_example_event()
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert results.keys() == ['hits', 'meta', 'aggregations']
assert results['meta'].keys() == ['timed_out']
assert results['meta']['timed_out'] is False
assert len(results['hits']) == 2
assert results['hits'][0].keys() == ['_score', '_type', '_id', '_source', '_index']
assert type(results['hits'][0]['_id']) == unicode
assert results['hits'][0]['_type'] == TMP_DOC_TYPE
assert results['hits'][0]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert results['hits'][0]['_source']['note'] == 'Example note'
assert results['hits'][0]['_source']['summary'] == 'Test Summary'
assert results['hits'][0]['_source']['type'] == 'event'
assert results['hits'][0]['_source']['details'].keys() == ['information']
assert results['hits'][0]['_source']['details']['information'] == 'Example information'
assert results['hits'][1].keys() == ['_score', '_type', '_id', '_source', '_index']
assert type(results['hits'][1]['_id']) == unicode
assert results['hits'][1]['_type'] == TMP_DOC_TYPE
assert results['hits'][1]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert results['hits'][1]['_source']['note'] == 'Example note'
assert results['hits'][1]['_source']['summary'] == 'Test Summary'
assert results['hits'][1]['_source']['type'] == 'event'
assert results['hits'][1]['_source']['details'].keys() == ['information']
assert results['hits'][1]['_source']['details']['information'] == 'Example information'
assert results['aggregations'].keys() == ['note']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 1
results['aggregations']['note']['terms'].sort()
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0]['key'] == 'Example note'
def test_simple_aggregation_source_field(self):
events = [
{
"test": "value",
"source": "somesource1"
},
{
"test": "value",
"source": "anothersource1"
},
{
"test": "value",
"source": "anothersource1"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"source": "somesource1 anothersource1"
},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('source'))
results = search_query.execute(self.es_client)
assert list(results['aggregations'].keys()) == ['source']
assert list(results['aggregations']['source'].keys()) == ['terms']
assert len(results['aggregations']['source']['terms']) == 3
assert list(results['aggregations']['source']['terms'][0].keys()) == [
'count', 'key'
]
assert results['aggregations']['source']['terms'][0]['count'] == 2
assert results['aggregations']['source']['terms'][0][
'key'] == 'anothersource1'
assert results['aggregations']['source']['terms'][1]['count'] == 1
assert results['aggregations']['source']['terms'][1][
'key'] == 'somesource1'
assert results['aggregations']['source']['terms'][2]['count'] == 1
assert results['aggregations']['source']['terms'][2][
'key'] == 'somesource1 anothersource1'
def test_simple_aggregation_note_field(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
{
"test": "value",
"note": "abvc space line"
},
]
for event in events:
self.populate_test_object(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('note'))
results = search_query.execute(self.es_client)
assert results['aggregations'].keys() == ['note']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 3
assert results['aggregations']['note']['terms'][0].keys() == [
'count', 'key'
]
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0]['key'] == 'abvc'
assert results['aggregations']['note']['terms'][1]['count'] == 1
assert results['aggregations']['note']['terms'][1][
'key'] == 'abvc space line'
assert results['aggregations']['note']['terms'][2]['count'] == 1
assert results['aggregations']['note']['terms'][2]['key'] == 'think'
def test_aggregation_multiple_layers(self):
events = [
{
"test": "value",
"details": {
"ip": "127.0.0.1"
},
},
{
"test": "value",
"details": {
"ip": "127.0.0.1"
},
},
{
"test": "value",
"details": {
"ip": "192.168.1.1"
},
},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('details.ip'))
results = search_query.execute(self.es_client)
assert list(results['aggregations'].keys()) == ['details.ip']
assert list(results['aggregations']['details.ip'].keys()) == ['terms']
assert len(results['aggregations']['details.ip']['terms']) == 2
assert results['aggregations']['details.ip']['terms'][0]['count'] == 2
assert results['aggregations']['details.ip']['terms'][0][
'key'] == "127.0.0.1"
assert results['aggregations']['details.ip']['terms'][1]['count'] == 1
assert results['aggregations']['details.ip']['terms'][1][
'key'] == "192.168.1.1"
def verify_events(options):
es_client = ElasticsearchClient(options.esservers)
for required_field in options.required_fields:
logger.debug('Looking for events without ' + required_field)
search_query = SearchQuery(hours=12)
search_query.add_must_not(ExistsMatch(required_field))
# Exclude all events that are mozdef related health and stats
search_query.add_must_not(TermMatch('_type', 'mozdefstats'))
search_query.add_must_not(TermMatch('_type', 'mozdefhealth'))
search_query.add_aggregation(Aggregation('_type'))
# We don't care about the actual events, we only want the numbers
results = search_query.execute(es_client, size=1)
for aggreg_term in results['aggregations']['_type']['terms']:
count = aggreg_term['count']
category = aggreg_term['key']
logger.error(
"Found {0} bad events of _type '{1}' missing '{2}' field".
format(count, category, required_field))
def esSearch(es):
search_query = SearchQuery(minutes=options.aggregationminutes)
search_query.add_aggregation(Aggregation('category'))
results = search_query.execute(es)
mozdefstats = dict(utctimestamp=toUTC(datetime.now()).isoformat())
mozdefstats['category'] = 'stats'
mozdefstats['hostname'] = socket.gethostname()
mozdefstats['mozdefhostname'] = mozdefstats['hostname']
mozdefstats['severity'] = 'INFO'
mozdefstats['source'] = 'mozdef'
mozdefstats['tags'] = ['mozdef', 'stats']
mozdefstats['summary'] = 'Aggregated category counts'
mozdefstats['processid'] = os.getpid()
mozdefstats['processname'] = sys.argv[0]
mozdefstats['details'] = dict(counts=list())
for bucket in results['aggregations']['category']['terms']:
entry = dict()
entry[bucket['key']] = bucket['count']
mozdefstats['details']['counts'].append(entry)
return mozdefstats
def test_multiple_values(self):
self.query.add_aggregation(Aggregation('note'))
self.query.add_aggregation(Aggregation('summary'))
assert self.query.aggregation == [
Aggregation('note'), Aggregation('summary')
]
def test_simple_input(self):
self.query.add_aggregation(Aggregation('note'))
assert self.query.aggregation == [Aggregation('note')]
def test_complex_aggregation_query_execute(self):
query = SearchQuery()
assert query.date_timedelta == {}
query.add_must(ExistsMatch('ip'))
query.add_aggregation(Aggregation('ip'))
self.populate_test_event({
'summary': 'Test Summary',
'ip': '127.0.0.1',
'details': {
'information': 'Example information'
}
})
self.populate_test_event({
'summary': 'Test Summary',
'ip': '1.2.3.4',
'details': {
'information': 'Example information'
}
})
self.populate_test_event({
'summary': 'Test Summary',
'ip': '1.2.3.4',
'details': {
'information': 'Example information'
}
})
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert sorted(results.keys()) == ['aggregations', 'hits', 'meta']
assert list(results['meta'].keys()) == ['timed_out']
assert results['meta']['timed_out'] is False
sorted_hits = sorted(results['hits'], key=lambda k: k['_source']['ip'])
assert len(sorted_hits) == 3
assert sorted(
sorted_hits[0].keys()) == ['_id', '_index', '_score', '_source']
assert type(sorted_hits[0]['_id']) == str
assert sorted_hits[0]['_index'] == datetime.now().strftime(
"events-%Y%m%d")
assert sorted_hits[0]['_source']['ip'] == '1.2.3.4'
assert sorted_hits[0]['_source']['summary'] == 'Test Summary'
assert sorted_hits[1]['_source']['type'] == 'event'
assert list(
sorted_hits[0]['_source']['details'].keys()) == ['information']
assert sorted_hits[0]['_source']['details'][
'information'] == 'Example information'
assert sorted(
sorted_hits[1].keys()) == ['_id', '_index', '_score', '_source']
assert type(sorted_hits[1]['_id']) == str
assert sorted_hits[1]['_index'] == datetime.now().strftime(
"events-%Y%m%d")
assert sorted_hits[1]['_source']['ip'] == '1.2.3.4'
assert sorted_hits[1]['_source']['summary'] == 'Test Summary'
assert sorted_hits[1]['_source']['type'] == 'event'
assert list(
sorted_hits[1]['_source']['details'].keys()) == ['information']
assert sorted_hits[1]['_source']['details'][
'information'] == 'Example information'
assert type(sorted_hits[2]['_id']) == str
assert sorted_hits[2]['_index'] == datetime.now().strftime(
"events-%Y%m%d")
assert sorted_hits[2]['_source']['ip'] == '127.0.0.1'
assert sorted_hits[2]['_source']['summary'] == 'Test Summary'
assert sorted_hits[2]['_source']['type'] == 'event'
assert list(
sorted_hits[2]['_source']['details'].keys()) == ['information']
assert sorted_hits[2]['_source']['details'][
'information'] == 'Example information'
assert list(results['aggregations'].keys()) == ['ip']
assert list(results['aggregations']['ip'].keys()) == ['terms']
assert len(results['aggregations']['ip']['terms']) == 2
assert results['aggregations']['ip']['terms'][0]['count'] == 2
assert results['aggregations']['ip']['terms'][0]['key'] == '1.2.3.4'
assert results['aggregations']['ip']['terms'][1]['count'] == 1
assert results['aggregations']['ip']['terms'][1]['key'] == '127.0.0.1'
def onMessage(self, request, response):
'''
request: http://bottlepy.org/docs/dev/api.html#the-request-object
response: http://bottlepy.org/docs/dev/api.html#the-response-object
'''
# an ES query/facet to count success/failed logins
# oriented to the data having
# category: authentication
# details.success marked true/false for success/failed auth
# details.username as the user
begindateUTC = None
enddateUTC = None
resultsList = list()
if begindateUTC is None:
begindateUTC = datetime.now() - timedelta(hours=12)
begindateUTC = toUTC(begindateUTC)
if enddateUTC is None:
enddateUTC = datetime.now()
enddateUTC = toUTC(enddateUTC)
es_client = ElasticsearchClient(
list('{0}'.format(s) for s in self.restoptions['esservers']))
search_query = SearchQuery()
# a query to tally users with failed logins
date_range_match = RangeMatch('utctimestamp', begindateUTC, enddateUTC)
search_query.add_must(date_range_match)
search_query.add_must(PhraseMatch('category', 'authentication'))
search_query.add_must(PhraseMatch('details.success', 'false'))
search_query.add_must(ExistsMatch('details.username'))
search_query.add_aggregation(Aggregation('details.success'))
search_query.add_aggregation(Aggregation('details.username'))
results = search_query.execute(es_client,
indices=['events', 'events-previous'])
# any usernames or words to ignore
# especially useful if ES is analyzing the username field and breaking apart [email protected]
# into user somewhere and .com
stoplist = self.options.ignoreusernames.split(',')
# walk the aggregate failed users
# and look for successes/failures
for t in results['aggregations']['details.username']['terms']:
if t['key'] in stoplist:
continue
failures = 0
success = 0
username = t['key']
details_query = SearchQuery()
details_query.add_must(date_range_match)
details_query.add_must(PhraseMatch('category', 'authentication'))
details_query.add_must(PhraseMatch('details.username', username))
details_query.add_aggregation(Aggregation('details.success'))
details_results = details_query.execute(es_client)
# details.success is boolean. As an aggregate is an int (0/1)
for details_term in details_results['aggregations'][
'details.success']['terms']:
if details_term['key'] == 1:
success = details_term['count']
if details_term['key'] == 0:
failures = details_term['count']
resultsList.append(
dict(username=username,
failures=failures,
success=success,
begin=begindateUTC.isoformat(),
end=enddateUTC.isoformat()))
response.body = json.dumps(resultsList)
response.status = 200
return (request, response)
