def test_aggregation_non_existing_layers_term(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
]
for event in events:
self.populate_test_event(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('details.ipinformation'))
results = search_query.execute(self.es_client)
assert results['aggregations'].keys() == ['details.ipinformation']
assert results['aggregations']['details.ipinformation'].keys() == [
'terms'
]
assert len(
results['aggregations']['details.ipinformation']['terms']) == 0
def esSearch(es):
search_query = SearchQuery(minutes=options.aggregationminutes)
search_query.add_aggregation(Aggregation('category'))
try:
results = search_query.execute(es)
mozdefstats = dict(utctimestamp=toUTC(datetime.now()).isoformat())
mozdefstats['category'] = 'stats'
mozdefstats['hostname'] = socket.gethostname()
mozdefstats['mozdefhostname'] = mozdefstats['hostname']
mozdefstats['severity'] = 'INFO'
mozdefstats['source'] = 'mozdef'
mozdefstats['tags'] = ['mozdef', 'stats']
mozdefstats['summary'] = 'Aggregated category counts'
mozdefstats['processid'] = os.getpid()
mozdefstats['processname'] = sys.argv[0]
mozdefstats['details'] = dict(counts=list())
for bucket in results['aggregations']['category']['terms']:
entry = dict()
entry[bucket['key']] = bucket['count']
mozdefstats['details']['counts'].append(entry)
return mozdefstats
except ElasticsearchBadServer:
logger.error(
'Elastic Search server could not be reached, check network connectivity'
)
def test_aggregation_non_existing_term(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
]
for event in events:
self.populate_test_object(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('example'))
results = search_query.execute(self.es_client)
assert results.keys() == ['hits', 'meta', 'aggregations']
assert len(results['hits']) == 4
assert results['aggregations'].keys() == ['example']
assert results['aggregations']['example'].keys() == ['terms']
assert results['aggregations']['example']['terms'] == []
def test_simple_aggregation_note_field(self):
events = [
{"test": "value", "note": "abvc"},
{"test": "value", "note": "abvc"},
{"test": "value", "note": "think"},
{"test": "value", "summary": "think"},
{"test": "value", "note": "abvc space line"},
]
for event in events:
self.populate_test_object(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('note'))
results = search_query.execute(self.es_client)
assert results['aggregations'].keys() == ['note']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 3
assert results['aggregations']['note']['terms'][0].keys() == ['count', 'key']
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0]['key'] == 'abvc'
assert results['aggregations']['note']['terms'][1]['count'] == 1
assert results['aggregations']['note']['terms'][1]['key'] == 'abvc space line'
assert results['aggregations']['note']['terms'][2]['count'] == 1
assert results['aggregations']['note']['terms'][2]['key'] == 'think'
def test_aggregation_multiple_layers(self):
events = [
{
"test": "value",
"details": {"ip": "127.0.0.1"},
},
{
"test": "value",
"details": {"ip": "127.0.0.1"},
},
{
"test": "value",
"details": {"ip": "192.168.1.1"},
},
]
for event in events:
self.populate_test_object(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('details.ip'))
results = search_query.execute(self.es_client)
assert results['aggregations'].keys() == ['details.ip']
assert results['aggregations']['details.ip'].keys() == ['terms']
assert len(results['aggregations']['details.ip']['terms']) == 2
assert results['aggregations']['details.ip']['terms'][0]['count'] == 2
assert results['aggregations']['details.ip']['terms'][0]['key'] == "127.0.0.1"
assert results['aggregations']['details.ip']['terms'][1]['count'] == 1
assert results['aggregations']['details.ip']['terms'][1]['key'] == "192.168.1.1"
def test_aggregation_without_must_fields(self):
event = self.generate_default_event()
event['_source']['utctimestamp'] = event['_source']['utctimestamp']()
self.populate_test_event(event)
search_query = SearchQuery(minutes=10)
search_query.add_aggregation(Aggregation('summary'))
results = search_query.execute(self.es_client)
assert results['aggregations']['summary']['terms'][0]['count'] == 1
def test_aggregation_with_aggregation_size(self):
for num in range(0, 100):
event = {'keyname': 'value' + str(num)}
self.populate_test_event(event)
search_query = SearchQuery()
search_query.add_must(ExistsMatch('keyname'))
search_query.add_aggregation(Aggregation('keyname', 2))
results = search_query.execute(self.es_client)
assert len(results['aggregations']['keyname']['terms']) == 2
def get_num_events(self):
self.flush('events')
search_query = SearchQuery()
search_query.add_must(TermMatch('_type', 'event'))
search_query.add_aggregation(Aggregation('_type'))
results = search_query.execute(self.es_client)
if len(results['aggregations']['_type']['terms']) != 0:
return results['aggregations']['_type']['terms'][0]['count']
else:
return 0
def test_multiple_aggregations(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
]
for event in events:
self.populate_test_event(event)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('note'))
search_query.add_aggregation(Aggregation('test'))
results = search_query.execute(self.es_client)
aggregation_keys = results['aggregations'].keys()
aggregation_keys.sort()
assert aggregation_keys == ['note', 'test']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 2
assert results['aggregations']['note']['terms'][0].keys() == [
'count', 'key'
]
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0]['key'] == 'abvc'
assert results['aggregations']['note']['terms'][1]['count'] == 1
assert results['aggregations']['note']['terms'][1]['key'] == 'think'
assert results['aggregations']['test'].keys() == ['terms']
assert len(results['aggregations']['test']['terms']) == 1
assert results['aggregations']['test']['terms'][0].keys() == [
'count', 'key'
]
assert results['aggregations']['test']['terms'][0]['count'] == 4
assert results['aggregations']['test']['terms'][0]['key'] == 'value'
def verify_events(options):
es_client = ElasticsearchClient(options.esservers)
for required_field in options.required_fields:
logger.debug('Looking for events without ' + required_field)
search_query = SearchQuery(hours=12)
search_query.add_must_not(ExistsMatch(required_field))
# Exclude all events that are mozdef related health and stats
search_query.add_must_not(TermMatch('_type', 'mozdefstats'))
search_query.add_must_not(TermMatch('_type', 'mozdefhealth'))
search_query.add_aggregation(Aggregation('_type'))
# We don't care about the actual events, we only want the numbers
results = search_query.execute(es_client, size=1)
for aggreg_term in results['aggregations']['_type']['terms']:
count = aggreg_term['count']
category = aggreg_term['key']
logger.error(
"Found {0} bad events of _type '{1}' missing '{2}' field".
format(count, category, required_field))
def esLdapResults(begindateUTC=None, enddateUTC=None):
'''an ES query/facet to count success/failed logins'''
resultsList = list()
if begindateUTC is None:
begindateUTC = datetime.now() - timedelta(hours=1)
begindateUTC = toUTC(begindateUTC)
if enddateUTC is None:
enddateUTC = datetime.now()
enddateUTC = toUTC(enddateUTC)
try:
es_client = ElasticsearchClient(
list('{0}'.format(s) for s in options.esservers))
search_query = SearchQuery()
range_match = RangeMatch('utctimestamp', begindateUTC, enddateUTC)
search_query.add_must(range_match)
search_query.add_must(TermMatch('tags', 'ldap'))
search_query.add_must(
TermMatch('details.result', 'LDAP_INVALID_CREDENTIALS'))
search_query.add_aggregation(Aggregation('details.result'))
search_query.add_aggregation(Aggregation('details.dn'))
results = search_query.execute(es_client, indices=['events'])
stoplist = ('o', 'mozilla', 'dc', 'com', 'mozilla.com',
'mozillafoundation.org', 'org', 'mozillafoundation')
for t in results['aggregations']['details.dn']['terms']:
if t['key'] in stoplist:
continue
failures = 0
success = 0
dn = t['key']
details_query = SearchQuery()
details_query.add_must(range_match)
details_query.add_must(TermMatch('tags', 'ldap'))
details_query.add_must(TermMatch('details.dn', dn))
details_query.add_aggregation(Aggregation('details.result'))
results = details_query.execute(es_client)
for t in results['aggregations']['details.result']['terms']:
if t['key'].upper() == 'LDAP_SUCCESS':
success = t['count']
if t['key'].upper() == 'LDAP_INVALID_CREDENTIALS':
failures = t['count']
resultsList.append(
dict(dn=dn,
failures=failures,
success=success,
begin=begindateUTC.isoformat(),
end=enddateUTC.isoformat()))
return (json.dumps(resultsList))
except Exception as e:
sys.stderr.write('Error trying to get ldap results: {0}\n'.format(e))
def test_aggregation_query_execute(self):
query = SearchQuery()
query.add_must(ExistsMatch('note'))
query.add_aggregation(Aggregation('note'))
assert query.date_timedelta == {}
self.populate_example_event()
self.populate_example_event()
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert results.keys() == ['hits', 'meta', 'aggregations']
assert results['meta'].keys() == ['timed_out']
assert results['meta']['timed_out'] is False
assert len(results['hits']) == 2
assert results['hits'][0].keys() == [
'_score', '_type', '_id', '_source', '_index'
]
assert type(results['hits'][0]['_id']) == unicode
assert results['hits'][0]['_type'] == 'event'
assert results['hits'][0]['_index'] == datetime.now().strftime(
"events-%Y%m%d")
assert results['hits'][0]['_source'].keys() == [
'note', 'details', 'summary'
]
assert results['hits'][0]['_source']['note'] == 'Example note'
assert results['hits'][0]['_source']['summary'] == 'Test Summary'
assert results['hits'][0]['_source']['details'].keys() == [
'information'
]
assert results['hits'][0]['_source']['details'][
'information'] == 'Example information'
assert results['hits'][1].keys() == [
'_score', '_type', '_id', '_source', '_index'
]
assert type(results['hits'][1]['_id']) == unicode
assert results['hits'][1]['_type'] == 'event'
assert results['hits'][1]['_index'] == datetime.now().strftime(
"events-%Y%m%d")
assert results['hits'][1]['_source'].keys() == [
'note', 'details', 'summary'
]
assert results['hits'][1]['_source']['note'] == 'Example note'
assert results['hits'][1]['_source']['summary'] == 'Test Summary'
assert results['hits'][1]['_source']['details'].keys() == [
'information'
]
assert results['hits'][1]['_source']['details'][
'information'] == 'Example information'
assert results['aggregations'].keys() == ['note']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 1
results['aggregations']['note']['terms'].sort()
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0][
'key'] == 'Example note'
def test_complex_aggregation_query_execute(self):
query = SearchQuery()
assert query.date_timedelta == {}
query.add_must(ExistsMatch('ip'))
query.add_aggregation(Aggregation('ip'))
self.populate_test_event({
'summary': 'Test Summary',
'ip': '127.0.0.1',
'details': {
'information': 'Example information'
}
})
self.populate_test_event({
'summary': 'Test Summary',
'ip': '1.2.3.4',
'details': {
'information': 'Example information'
}
})
self.populate_test_event({
'summary': 'Test Summary',
'ip': '1.2.3.4',
'details': {
'information': 'Example information'
}
})
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert results.keys() == ['hits', 'meta', 'aggregations']
assert results['meta'].keys() == ['timed_out']
assert results['meta']['timed_out'] is False
sorted_hits = sorted(results['hits'], key=lambda k: k['_source']['ip'])
assert len(sorted_hits) == 3
assert sorted_hits[0].keys() == [
'_score', '_type', '_id', '_source', '_index'
]
assert type(sorted_hits[0]['_id']) == unicode
assert sorted_hits[0]['_type'] == 'event'
assert sorted_hits[0]['_index'] == datetime.now().strftime(
"events-%Y%m%d")
assert sorted_hits[0]['_source'].keys() == ['ip', 'details', 'summary']
assert sorted_hits[0]['_source']['ip'] == '1.2.3.4'
assert sorted_hits[0]['_source']['summary'] == 'Test Summary'
assert sorted_hits[0]['_source']['details'].keys() == ['information']
assert sorted_hits[0]['_source']['details'][
'information'] == 'Example information'
assert sorted_hits[1].keys() == [
'_score', '_type', '_id', '_source', '_index'
]
assert type(sorted_hits[1]['_id']) == unicode
assert sorted_hits[1]['_type'] == 'event'
assert sorted_hits[1]['_index'] == datetime.now().strftime(
"events-%Y%m%d")
assert sorted_hits[1]['_source'].keys() == ['ip', 'details', 'summary']
assert sorted_hits[1]['_source']['ip'] == '1.2.3.4'
assert sorted_hits[1]['_source']['summary'] == 'Test Summary'
assert sorted_hits[1]['_source']['details'].keys() == ['information']
assert sorted_hits[1]['_source']['details'][
'information'] == 'Example information'
assert sorted_hits[2].keys() == [
'_score', '_type', '_id', '_source', '_index'
]
assert type(sorted_hits[2]['_id']) == unicode
assert sorted_hits[2]['_type'] == 'event'
assert sorted_hits[2]['_index'] == datetime.now().strftime(
"events-%Y%m%d")
assert sorted_hits[2]['_source'].keys() == ['ip', 'details', 'summary']
assert sorted_hits[2]['_source']['ip'] == '127.0.0.1'
assert sorted_hits[2]['_source']['summary'] == 'Test Summary'
assert sorted_hits[2]['_source']['details'].keys() == ['information']
assert sorted_hits[2]['_source']['details'][
'information'] == 'Example information'
assert results['aggregations'].keys() == ['ip']
assert results['aggregations']['ip'].keys() == ['terms']
assert len(results['aggregations']['ip']['terms']) == 2
results['aggregations']['ip']['terms'].sort()
assert results['aggregations']['ip']['terms'][0]['count'] == 1
assert results['aggregations']['ip']['terms'][0]['key'] == '127.0.0.1'
assert results['aggregations']['ip']['terms'][1]['count'] == 2
assert results['aggregations']['ip']['terms'][1]['key'] == '1.2.3.4'