def esLdapResults(begindateUTC=None, enddateUTC=None):
'''an ES query/facet to count success/failed logins'''
resultsList = list()
if begindateUTC is None:
begindateUTC = datetime.now() - timedelta(hours=1)
begindateUTC = toUTC(begindateUTC)
if enddateUTC is None:
enddateUTC = datetime.now()
enddateUTC = toUTC(enddateUTC)
try:
es_client = ElasticsearchClient(list('{0}'.format(s) for s in options.esservers))
search_query = SearchQuery()
range_match = RangeMatch('utctimestamp', begindateUTC, enddateUTC)
search_query.add_must(range_match)
search_query.add_must(TermMatch('tags', 'ldap'))
search_query.add_must(TermMatch('details.result', 'LDAP_INVALID_CREDENTIALS'))
search_query.add_aggregation(Aggregation('details.result'))
search_query.add_aggregation(Aggregation('details.dn'))
results = search_query.execute(es_client, indices=['events'])
stoplist = ('o', 'mozilla', 'dc', 'com', 'mozilla.com', 'mozillafoundation.org', 'org', 'mozillafoundation')
for t in results['aggregations']['details.dn']['terms']:
if t['key'] in stoplist:
continue
failures = 0
success = 0
dn = t['key']
details_query = SearchQuery()
details_query.add_must(range_match)
details_query.add_must(TermMatch('tags', 'ldap'))
details_query.add_must(TermMatch('details.dn', dn))
details_query.add_aggregation(Aggregation('details.result'))
results = details_query.execute(es_client)
for t in results['aggregations']['details.result']['terms']:
if t['key'].upper() == 'LDAP_SUCCESS':
success = t['count']
if t['key'].upper() == 'LDAP_INVALID_CREDENTIALS':
failures = t['count']
resultsList.append(
dict(
dn=dn,
failures=failures,
success=success,
begin=begindateUTC.isoformat(),
end=enddateUTC.isoformat()
)
)
return(json.dumps(resultsList))
except Exception as e:
sys.stderr.write('Error trying to get ldap results: {0}\n'.format(e))
def test_simple_query_execute(self):
query = SearchQuery()
query.add_must(ExistsMatch('note'))
assert query.date_timedelta == {}
self.populate_example_event()
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert results.keys() == ['hits', 'meta']
assert results['meta'].keys() == ['timed_out']
assert results['meta']['timed_out'] is False
assert len(results['hits']) == 1
assert results['hits'][0].keys() == ['_score', '_type', '_id', '_source', '_index']
assert type(results['hits'][0]['_id']) == unicode
assert results['hits'][0]['_type'] == 'event'
assert results['hits'][0]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert results['hits'][0]['_source']['note'] == 'Example note'
assert results['hits'][0]['_source']['summary'] == 'Test Summary'
assert results['hits'][0]['_source']['details'].keys() == ['information']
assert results['hits'][0]['_source']['details']['information'] == 'Example information'
with pytest.raises(KeyError):
results['abcdefg']
with pytest.raises(KeyError):
results['abcdefg']['test']
def esSearch(es, macassignments=None):
'''
Search ES for an event that ties a username to a mac address
This example searches for junos wifi correlations on authentication success
Expecting an event like: user: [email protected]; mac: 5c:f9:38:b1:de:cf; author reason: roamed session; ssid: ANSSID; AP 46/2\n
'''
usermacre=re.compile(r'''user: (?P<username>.*?); mac: (?P<macaddress>.*?); ''',re.IGNORECASE)
correlations={}
search_query = SearchQuery(minutes=options.correlationminutes)
search_query.add_must(TermMatch('details.program', 'AUTHORIZATION-SUCCESS'))
search_query.add_must_not(PhraseMatch('summary', 'last-resort'))
try:
full_results = search_query.execute(es)
results = full_results['hits']
for r in results:
fields = re.search(usermacre,r['_source']['summary'])
if fields:
if '{0} {1}'.format(fields.group('username'),fields.group('macaddress')) not in correlations:
if fields.group('macaddress')[0:8].lower() in macassignments:
entity=macassignments[fields.group('macaddress')[0:8].lower()]
else:
entity='unknown'
correlations['{0} {1}'.format(fields.group('username'),fields.group('macaddress'))]=dict(username=fields.group('username'),
macaddress=fields.group('macaddress'),
entity=entity,
utctimestamp=r['_source']['utctimestamp'])
return correlations
except ElasticsearchBadServer:
logger.error('Elastic Search server could not be reached, check network connectivity')
def test_aggregation_multiple_layers(self):
events = [
{
"test": "value",
"details": {"ip": "127.0.0.1"},
},
{
"test": "value",
"details": {"ip": "127.0.0.1"},
},
{
"test": "value",
"details": {"ip": "192.168.1.1"},
},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('details.ip'))
results = search_query.execute(self.es_client)
assert results['aggregations'].keys() == ['details.ip']
assert results['aggregations']['details.ip'].keys() == ['terms']
assert len(results['aggregations']['details.ip']['terms']) == 2
assert results['aggregations']['details.ip']['terms'][0]['count'] == 2
assert results['aggregations']['details.ip']['terms'][0]['key'] == "127.0.0.1"
assert results['aggregations']['details.ip']['terms'][1]['count'] == 1
assert results['aggregations']['details.ip']['terms'][1]['key'] == "192.168.1.1"
def test_writing_event_defaults(self):
query = SearchQuery()
default_event = {}
self.populate_test_event(default_event)
self.refresh(self.event_index_name)
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
assert len(results['hits']) == 1
assert sorted(results['hits'][0].keys()) == [
'_id', '_index', '_score', '_source', '_type'
]
saved_event = results['hits'][0]['_source']
assert 'category' in saved_event
assert 'details' in saved_event
assert 'hostname' in saved_event
assert 'mozdefhostname' in saved_event
assert 'processid' in saved_event
assert 'processname' in saved_event
assert 'receivedtimestamp' in saved_event
assert 'severity' in saved_event
assert 'source' in saved_event
assert 'summary' in saved_event
assert 'tags' in saved_event
assert 'timestamp' in saved_event
assert 'utctimestamp' in saved_event
assert 'category' in saved_event
def test_writing_with_type(self):
query = SearchQuery()
default_event = {
"_type": "example",
"_source": {
"receivedtimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
}
self.populate_test_event(default_event)
self.refresh(self.event_index_name)
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
assert len(results['hits']) == 1
assert sorted(results['hits'][0].keys()) == [
'_id', '_index', '_score', '_source', '_type'
]
assert results['hits'][0]['_type'] == 'example'
assert results['hits'][0]['_source']['summary'] == 'Test summary'
assert results['hits'][0]['_source']['details'] == {
"note": "Example note"
}
def test_simple_query_execute(self):
query = SearchQuery()
query.add_must(ExistsMatch('note'))
assert query.date_timedelta == {}
self.populate_example_event()
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert sorted(results.keys()) == ['hits', 'meta']
assert list(results['meta'].keys()) == ['timed_out']
assert results['meta']['timed_out'] is False
assert len(results['hits']) == 1
assert sorted(results['hits'][0].keys()) == ['_id', '_index', '_score', '_source']
assert type(results['hits'][0]['_id']) == str
assert results['hits'][0]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert results['hits'][0]['_source']['note'] == 'Example note'
assert results['hits'][0]['_source']['summary'] == 'Test Summary'
assert results['hits'][0]['_source']['type'] == 'event'
assert list(results['hits'][0]['_source']['details'].keys()) == ['information']
assert results['hits'][0]['_source']['details']['information'] == 'Example information'
with pytest.raises(KeyError):
results['abcdefg']
with pytest.raises(KeyError):
results['abcdefg']['test']
def test_without_time_defined(self):
query = SearchQuery()
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {}
default_event = {
"utctimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(default_event)
default_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'days': 11})
default_event[
'receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'days': 11})
self.populate_test_event(default_event)
not_old_event = default_event
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'days': 9})
not_old_event[
'receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'days': 9})
self.populate_test_event(not_old_event)
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 3
def kibanaDashboards():
resultsList = []
try:
es_client = ElasticsearchClient(
(list('{0}'.format(s) for s in options.esservers)))
search_query = SearchQuery()
search_query.add_must(TermMatch('type', 'dashboard'))
results = search_query.execute(es_client, indices=['.kibana'])
for dashboard in results['hits']:
dashboard_id = dashboard['_id']
if dashboard_id.startswith('dashboard:'):
dashboard_id = dashboard_id.replace('dashboard:', '')
resultsList.append({
'name':
dashboard['_source']['dashboard']['title'],
'id':
dashboard_id
})
except ElasticsearchInvalidIndex as e:
logger.error('Kibana dashboard index not found: {0}\n'.format(e))
except Exception as e:
logger.error('Kibana dashboard received error: {0}\n'.format(e))
return json.dumps(resultsList)
def test_without_time_defined(self):
query = SearchQuery()
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {}
default_event = {
"utctimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(default_event)
default_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'days': 11})
default_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'days': 11})
self.populate_test_event(default_event)
not_old_event = default_event
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'days': 9})
not_old_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'days': 9})
self.populate_test_event(not_old_event)
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 3
def test_aggregation_non_existing_term(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('example'))
results = search_query.execute(self.es_client)
assert sorted(results.keys()) == ['aggregations', 'hits', 'meta']
assert len(results['hits']) == 4
assert list(results['aggregations'].keys()) == ['example']
assert list(results['aggregations']['example'].keys()) == ['terms']
assert results['aggregations']['example']['terms'] == []
def getESAlerts(es):
search_query = SearchQuery(minutes=50)
# We use an ExistsMatch here just to satisfy the
# requirements of a search query must have some "Matchers"
search_query.add_must(ExistsMatch('summary'))
results = search_query.execute(es, indices=['alerts'], size=10000)
return results
def wrapper(query: SearchQuery, esindex: str) -> Optional[Entry]:
results = query.execute(client, indices=[esindex]).get('hits', [])
if len(results) == 0:
return None
state_dict = results[0].get('_source', {})
try:
state_dict['localities'] = [
# Convert dictionary localities into `Locality`s after
# parsing the `datetime` from `lastaction`.
Locality(**_dict_take(
{
k: v if k != 'lastaction' else toUTC(v)
for k, v in loc.items()
}, Locality._fields)) for loc in state_dict['localities']
]
eid = results[0]['_id']
state = State(**_dict_take(state_dict, State._fields))
return Entry(eid, state)
except TypeError:
return None
except KeyError:
return None
def test_simple_aggregation_note_field(self):
events = [
{"test": "value", "note": "abvc"},
{"test": "value", "note": "abvc"},
{"test": "value", "note": "think"},
{"test": "value", "summary": "think"},
{"test": "value", "note": "abvc space line"},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('note'))
results = search_query.execute(self.es_client)
assert results['aggregations'].keys() == ['note']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 3
assert results['aggregations']['note']['terms'][0].keys() == ['count', 'key']
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0]['key'] == 'abvc'
assert results['aggregations']['note']['terms'][1]['count'] == 1
assert results['aggregations']['note']['terms'][1]['key'] == 'abvc space line'
assert results['aggregations']['note']['terms'][2]['count'] == 1
assert results['aggregations']['note']['terms'][2]['key'] == 'think'
def test_beginning_time_seconds_received_timestamp(self):
query = SearchQuery(seconds=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'seconds': 10}
default_event = {
"receivedtimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(default_event)
too_old_event = default_event
too_old_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 11})
too_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 11})
self.populate_test_event(too_old_event)
not_old_event = default_event
not_old_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 9})
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 9})
self.populate_test_event(not_old_event)
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 2
def test_beginning_time_seconds_received_timestamp(self):
query = SearchQuery(seconds=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'seconds': 10}
default_event = {
"receivedtimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(default_event)
too_old_event = default_event
too_old_event[
'receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'seconds': 11})
too_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'seconds': 11})
self.populate_test_event(too_old_event)
not_old_event = default_event
not_old_event[
'receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'seconds': 9})
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'seconds': 9})
self.populate_test_event(not_old_event)
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 2
def test_writing_event_defaults(self):
query = SearchQuery()
default_event = {}
self.populate_test_event(default_event)
self.refresh(self.event_index_name)
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
assert len(results['hits']) == 1
assert sorted(results['hits'][0].keys()) == ['_id', '_index', '_score', '_source', '_type']
saved_event = results['hits'][0]['_source']
assert 'category' in saved_event
assert 'details' in saved_event
assert 'hostname' in saved_event
assert 'mozdefhostname' in saved_event
assert 'processid' in saved_event
assert 'processname' in saved_event
assert 'receivedtimestamp' in saved_event
assert 'severity' in saved_event
assert 'source' in saved_event
assert 'summary' in saved_event
assert 'tags' in saved_event
assert 'timestamp' in saved_event
assert 'utctimestamp' in saved_event
assert 'category' in saved_event
def getESAlerts(es):
search_query = SearchQuery(minutes=50)
# We use an ExistsMatch here just to satisfy the
# requirements of a search query must have some "Matchers"
search_query.add_must(ExistsMatch('summary'))
results = search_query.execute(es, indices=['alerts'], size=10000)
return results
def kibanaDashboards():
resultsList = []
try:
es_client = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)))
search_query = SearchQuery()
search_query.add_must(TermMatch('_type', 'dashboard'))
results = search_query.execute(es_client, indices=['.kibana'])
for dashboard in results['hits']:
resultsList.append({
'name': dashboard['_source']['title'],
'url': "%s#/%s/%s" % (
options.kibanaurl,
"dashboard",
dashboard['_id']
)
})
except ElasticsearchInvalidIndex as e:
sys.stderr.write('Kibana dashboard index not found: {0}\n'.format(e))
except Exception as e:
sys.stderr.write('Kibana dashboard received error: {0}\n'.format(e))
return json.dumps(resultsList)
def test_aggregation_non_existing_layers_term(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('details.ipinformation'))
results = search_query.execute(self.es_client)
assert results['aggregations'].keys() == ['details.ipinformation']
assert results['aggregations']['details.ipinformation'].keys() == [
'terms'
]
assert len(
results['aggregations']['details.ipinformation']['terms']) == 0
def search_and_verify_event(self, expected_event):
self.refresh('events')
search_query = SearchQuery(minutes=5)
search_query.add_must(ExistsMatch('tags'))
results = search_query.execute(self.es_client)
assert len(results['hits']) == 1
saved_event = results['hits'][0]['_source']
self.verify_event(saved_event, expected_event)
def search_and_verify_event(self, expected_event):
self.refresh('events')
search_query = SearchQuery(minutes=5)
search_query.add_must(ExistsMatch('tags'))
results = search_query.execute(self.es_client)
assert len(results['hits']) == 1
saved_event = results['hits'][0]['_source']
self.verify_event(saved_event, expected_event)
def test_execute_without_size(self):
for num in range(0, 1200):
self.populate_example_event()
self.refresh(self.event_index_name)
query = SearchQuery()
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
assert len(results['hits']) == 1000
def test_execute_without_size(self):
for num in range(0, 1200):
self.populate_example_event()
self.refresh(self.event_index_name)
query = SearchQuery()
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
assert len(results['hits']) == 1000
def getSqsStats(es):
search_query = SearchQuery(minutes=15)
search_query.add_must([
TermMatch('type', 'mozdefhealth'),
TermMatch('category', 'mozdef'),
TermMatch('tags', 'sqs-latest'),
])
results = search_query.execute(es, indices=['mozdefstate'])
return results['hits']
def get_num_events(self):
self.refresh('events')
search_query = SearchQuery()
search_query.add_must(TermMatch('_type', 'event'))
search_query.add_aggregation(Aggregation('_type'))
results = search_query.execute(self.es_client)
if len(results['aggregations']['_type']['terms']) != 0:
return results['aggregations']['_type']['terms'][0]['count']
else:
return 0
def get_num_events(self):
self.refresh('events')
search_query = SearchQuery()
search_query.add_must(TermMatch('_type', 'event'))
search_query.add_aggregation(Aggregation('_type'))
results = search_query.execute(self.es_client)
if len(results['aggregations']['_type']['terms']) != 0:
return results['aggregations']['_type']['terms'][0]['count']
else:
return 0
def getSqsStats(es):
search_query = SearchQuery(minutes=15)
search_query.add_must([
TermMatch('_type', 'mozdefhealth'),
TermMatch('category', 'mozdef'),
TermMatch('tags', 'sqs-latest'),
])
results = search_query.execute(es, indices=['mozdefstate'])
return results['hits']
def test_aggregation_with_aggregation_size(self):
for num in range(0, 100):
event = {'keyname': 'value' + str(num)}
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(ExistsMatch('keyname'))
search_query.add_aggregation(Aggregation('keyname', 2))
results = search_query.execute(self.es_client)
assert len(results['aggregations']['keyname']['terms']) == 2
def test_aggregation_with_aggregation_size(self):
for num in range(0, 100):
event = {'keyname': 'value' + str(num)}
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(ExistsMatch('keyname'))
search_query.add_aggregation(Aggregation('keyname', 2))
results = search_query.execute(self.es_client)
assert len(results['aggregations']['keyname']['terms']) == 2
def test_query_class(self):
for query, events in self.query_tests().iteritems():
for event in events:
if pytest.config.option.delete_indexes:
self.reset_elasticsearch()
self.setup_elasticsearch()
self.populate_test_object(event)
self.flush(self.event_index_name)
# Testing must
search_query = SearchQuery()
search_query.add_must(query)
query_result = search_query.execute(self.es_client)
self.verify_test(query_result, self.positive_test)
# Testing must_not
search_query = SearchQuery()
search_query.add_must_not(query)
query_result = search_query.execute(self.es_client)
self.verify_test(query_result, self.positive_test is False)
def test_query_class(self):
for query, events in self.query_tests().iteritems():
for event in events:
if pytest.config.option.delete_indexes:
self.reset_elasticsearch()
self.setup_elasticsearch()
self.populate_test_object(event)
self.refresh(self.event_index_name)
# Testing must
search_query = SearchQuery()
search_query.add_must(query)
query_result = search_query.execute(self.es_client)
self.verify_test(query_result, self.positive_test)
# Testing must_not
search_query = SearchQuery()
search_query.add_must_not(query)
query_result = search_query.execute(self.es_client)
self.verify_test(query_result, self.positive_test is False)
def test_multiple_aggregations(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('note'))
search_query.add_aggregation(Aggregation('test'))
results = search_query.execute(self.es_client)
aggregation_keys = results['aggregations'].keys()
aggregation_keys.sort()
assert aggregation_keys == ['note', 'test']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 2
assert results['aggregations']['note']['terms'][0].keys() == [
'count', 'key'
]
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0]['key'] == 'abvc'
assert results['aggregations']['note']['terms'][1]['count'] == 1
assert results['aggregations']['note']['terms'][1]['key'] == 'think'
assert results['aggregations']['test'].keys() == ['terms']
assert len(results['aggregations']['test']['terms']) == 1
assert results['aggregations']['test']['terms'][0].keys() == [
'count', 'key'
]
assert results['aggregations']['test']['terms'][0]['count'] == 4
assert results['aggregations']['test']['terms'][0]['key'] == 'value'
def test_aggregation_without_must_fields(self):
event = self.generate_default_event()
event['_source']['utctimestamp'] = event['_source']['utctimestamp']()
event['_source']['receivedtimestamp'] = event['_source']['receivedtimestamp']()
self.populate_test_event(event)
self.refresh(self.event_index_name)
search_query = SearchQuery(minutes=10)
search_query.add_aggregation(Aggregation('source'))
results = search_query.execute(self.es_client)
assert results['aggregations']['source']['terms'][0]['count'] == 1
def test_aggregation_without_must_fields(self):
event = self.generate_default_event()
event['_source']['utctimestamp'] = event['_source']['utctimestamp']()
event['_source']['receivedtimestamp'] = event['_source']['receivedtimestamp']()
self.populate_test_event(event)
self.refresh(self.event_index_name)
search_query = SearchQuery(minutes=10)
search_query.add_aggregation(Aggregation('source'))
results = search_query.execute(self.es_client)
assert results['aggregations']['source']['terms'][0]['count'] == 1
def test_aggregation_query_execute(self):
query = SearchQuery()
query.add_must(ExistsMatch('note'))
query.add_aggregation(Aggregation('note'))
assert query.date_timedelta == {}
self.populate_example_event()
self.populate_example_event()
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert results.keys() == ['hits', 'meta', 'aggregations']
assert results['meta'].keys() == ['timed_out']
assert results['meta']['timed_out'] is False
assert len(results['hits']) == 2
assert results['hits'][0].keys() == ['_score', '_type', '_id', '_source', '_index']
assert type(results['hits'][0]['_id']) == unicode
assert results['hits'][0]['_type'] == TMP_DOC_TYPE
assert results['hits'][0]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert results['hits'][0]['_source']['note'] == 'Example note'
assert results['hits'][0]['_source']['summary'] == 'Test Summary'
assert results['hits'][0]['_source']['type'] == 'event'
assert results['hits'][0]['_source']['details'].keys() == ['information']
assert results['hits'][0]['_source']['details']['information'] == 'Example information'
assert results['hits'][1].keys() == ['_score', '_type', '_id', '_source', '_index']
assert type(results['hits'][1]['_id']) == unicode
assert results['hits'][1]['_type'] == TMP_DOC_TYPE
assert results['hits'][1]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert results['hits'][1]['_source']['note'] == 'Example note'
assert results['hits'][1]['_source']['summary'] == 'Test Summary'
assert results['hits'][1]['_source']['type'] == 'event'
assert results['hits'][1]['_source']['details'].keys() == ['information']
assert results['hits'][1]['_source']['details']['information'] == 'Example information'
assert results['aggregations'].keys() == ['note']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 1
results['aggregations']['note']['terms'].sort()
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0]['key'] == 'Example note'
def test_simple_aggregation_source_field(self):
events = [
{
"test": "value",
"source": "somesource1"
},
{
"test": "value",
"source": "anothersource1"
},
{
"test": "value",
"source": "anothersource1"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"source": "somesource1 anothersource1"
},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('source'))
results = search_query.execute(self.es_client)
assert list(results['aggregations'].keys()) == ['source']
assert list(results['aggregations']['source'].keys()) == ['terms']
assert len(results['aggregations']['source']['terms']) == 3
assert list(results['aggregations']['source']['terms'][0].keys()) == [
'count', 'key'
]
assert results['aggregations']['source']['terms'][0]['count'] == 2
assert results['aggregations']['source']['terms'][0][
'key'] == 'anothersource1'
assert results['aggregations']['source']['terms'][1]['count'] == 1
assert results['aggregations']['source']['terms'][1][
'key'] == 'somesource1'
assert results['aggregations']['source']['terms'][2]['count'] == 1
assert results['aggregations']['source']['terms'][2][
'key'] == 'somesource1 anothersource1'
def test_query_class(self):
for testcase in self.query_tests():
query = testcase[0]
events = testcase[1]
for event in events:
if self.config_delete_indexes:
self.reset_elasticsearch()
self.setup_elasticsearch()
self.populate_test_object(event)
self.refresh(self.event_index_name)
# Testing must
search_query = SearchQuery()
search_query.add_must(query)
query_result = search_query.execute(self.es_client)
self.verify_test(query_result, self.positive_test)
# Testing must_not
search_query = SearchQuery()
search_query.add_must_not(query)
query_result = search_query.execute(self.es_client)
self.verify_test(query_result, self.positive_test is False)
def test_simple_aggregation_note_field(self):
events = [
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "abvc"
},
{
"test": "value",
"note": "think"
},
{
"test": "value",
"summary": "think"
},
{
"test": "value",
"note": "abvc space line"
},
]
for event in events:
self.populate_test_object(event)
self.flush(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('note'))
results = search_query.execute(self.es_client)
assert results['aggregations'].keys() == ['note']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 3
assert results['aggregations']['note']['terms'][0].keys() == [
'count', 'key'
]
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0]['key'] == 'abvc'
assert results['aggregations']['note']['terms'][1]['count'] == 1
assert results['aggregations']['note']['terms'][1][
'key'] == 'abvc space line'
assert results['aggregations']['note']['terms'][2]['count'] == 1
assert results['aggregations']['note']['terms'][2]['key'] == 'think'
def test_writing_dot_fieldname(self):
event = json.dumps({"key.othername": "example value for string of json test"})
self.es_client.save_event(body=event)
self.refresh(self.event_index_name)
num_events = self.get_num_events()
assert num_events == 1
query = SearchQuery()
query.add_must(ExistsMatch('key.othername'))
results = query.execute(self.es_client)
assert sorted(results['hits'][0].keys()) == ['_id', '_index', '_score', '_source', '_type']
assert results['hits'][0]['_source']['key.othername'] == 'example value for string of json test'
assert len(results['hits']) == 1
assert results['hits'][0]['_type'] == 'event'
def wrapper(esindex: Index=None) -> Optional[Record]:
query = SearchQuery()
query.add_must(TermMatch('type_', _TYPE_NAME))
results = query.execute(client, indices=[esindex])
if len(results['hits']) == 0:
return None
eid = results['hits'][0]['_id']
state = ExecutionState(**_dict_take(
results['hits'][0].get('_source', {}),
ExecutionState._fields))
return Record(eid, state)
def test_time_received_timestamp(self):
query = SearchQuery(seconds=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'seconds': 10}
received_timestamp_default_event = {
"receivedtimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(received_timestamp_default_event)
utctimestamp_default_event = {
"utctimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(utctimestamp_default_event)
default_event = {
"utctimestamp": UnitTestSuite.current_timestamp(),
"receivedtimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(default_event)
modified_received_timestamp_event = default_event
modified_received_timestamp_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 11})
self.populate_test_event(modified_received_timestamp_event)
modified_utc_timestamp_event = default_event
modified_utc_timestamp_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 9})
self.populate_test_event(modified_utc_timestamp_event)
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 5
def searchESForBROAttackers(es, threshold):
search_query = SearchQuery(hours=2)
search_query.add_must([
PhraseMatch('category', 'bro'),
PhraseMatch('details.note', 'MozillaHTTPErrors::Excessive_HTTP_Errors_Attacker')
])
full_results = search_query.execute(es)
results = full_results['hits']
# Hit count is buried in the 'sub' field
# as: 'sub': u'6 in 1.0 hr, eps: 0'
# cull the records for hitcounts over the threshold before returning
attackers = list()
for r in results:
hitcount = int(r['_source']['details']['sub'].split()[0])
if hitcount > threshold:
attackers.append(r)
return attackers
def test_without_utctimestamp(self):
query = SearchQuery(days=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'days': 10}
default_event = {
"timestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_object(default_event)
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 0
def searchESForBROAttackers(es, threshold):
search_query = SearchQuery(hours=2)
search_query.add_must([
PhraseMatch('category', 'bronotice'),
PhraseMatch('details.note', 'MozillaHTTPErrors::Excessive_HTTP_Errors_Attacker')
])
full_results = search_query.execute(es)
results = full_results['hits']
# Hit count is buried in the 'sub' field
# as: 'sub': u'6 in 1.0 hr, eps: 0'
# cull the records for hitcounts over the threshold before returning
attackers = list()
for r in results:
hitcount = int(r['_source']['details']['sub'].split()[0])
if hitcount > threshold:
attackers.append(r)
return attackers
def test_without_utctimestamp(self):
query = SearchQuery(days=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'days': 10}
default_event = {
"timestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_object(default_event)
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 0
def onMessage(self, message):
hostname = _most_common_hostname(message.get('events', []))
query = SearchQuery(hours=self._config.search_window_hours)
query.add_must([
TermMatch('category', 'syslog'),
TermMatch('hostname', hostname),
TermMatch('details.program', 'sshd'),
PhraseMatch('summary', 'Accepted publickey for '),
])
results = query.execute(self._es_client,
indices=self._config.indices_to_search)
events = [hit.get('_source', {}) for hit in results.get('hits', [])]
return enrich(message, events)
def test_aggregation_non_existing_layers_term(self):
events = [
{"test": "value", "note": "abvc"},
{"test": "value", "note": "abvc"},
{"test": "value", "note": "think"},
{"test": "value", "summary": "think"},
]
for event in events:
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(TermMatch('test', 'value'))
search_query.add_aggregation(Aggregation('details.ipinformation'))
results = search_query.execute(self.es_client)
assert results['aggregations'].keys() == ['details.ipinformation']
assert results['aggregations']['details.ipinformation'].keys() == ['terms']
assert len(results['aggregations']['details.ipinformation']['terms']) == 0
def test_writing_with_source(self):
query = SearchQuery()
default_event = {
"_source": {
"receivedtimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
}
self.populate_test_event(default_event)
self.refresh(self.event_index_name)
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
assert len(results['hits']) == 1
assert sorted(results['hits'][0].keys()) == ['_id', '_index', '_score', '_source', '_type']
assert results['hits'][0]['_type'] == 'event'
def esSearch(es):
search_query = SearchQuery(minutes=options.aggregationminutes)
search_query.add_aggregation(Aggregation('category'))
results = search_query.execute(es)
mozdefstats = dict(utctimestamp=toUTC(datetime.now()).isoformat())
mozdefstats['category'] = 'stats'
mozdefstats['hostname'] = socket.gethostname()
mozdefstats['mozdefhostname'] = mozdefstats['hostname']
mozdefstats['severity'] = 'INFO'
mozdefstats['source'] = 'mozdef'
mozdefstats['tags'] = ['mozdef', 'stats']
mozdefstats['summary'] = 'Aggregated category counts'
mozdefstats['processid'] = os.getpid()
mozdefstats['processname'] = sys.argv[0]
mozdefstats['details'] = dict(counts=list())
for bucket in results['aggregations']['category']['terms']:
entry = dict()
entry[bucket['key']] = bucket['count']
mozdefstats['details']['counts'].append(entry)
return mozdefstats
def verify_events(options):
es_client = ElasticsearchClient(options.esservers)
for required_field in options.required_fields:
logger.debug('Looking for events without ' + required_field)
search_query = SearchQuery(hours=12)
search_query.add_must_not(ExistsMatch(required_field))
# Exclude all events that are mozdef related health and stats
search_query.add_must_not(TermMatch('_type', 'mozdefstats'))
search_query.add_must_not(TermMatch('_type', 'mozdefhealth'))
search_query.add_aggregation(Aggregation('_type'))
# We don't care about the actual events, we only want the numbers
results = search_query.execute(es_client, size=1)
for aggreg_term in results['aggregations']['_type']['terms']:
count = aggreg_term['count']
category = aggreg_term['key']
logger.error("Found {0} bad events of _type '{1}' missing '{2}' field".format(
count,
category,
required_field
))
def onMessage(self, request, response):
'''
request: http://bottlepy.org/docs/dev/api.html#the-request-object
response: http://bottlepy.org/docs/dev/api.html#the-response-object
'''
# an ES query/facet to count success/failed logins
# oriented to the data having
# category: authentication
# details.success marked true/false for success/failed auth
# details.username as the user
begindateUTC=None
enddateUTC=None
resultsList = list()
if begindateUTC is None:
begindateUTC = datetime.now() - timedelta(hours=12)
begindateUTC = toUTC(begindateUTC)
if enddateUTC is None:
enddateUTC = datetime.now()
enddateUTC = toUTC(enddateUTC)
es_client = ElasticsearchClient(list('{0}'.format(s) for s in self.restoptions['esservers']))
search_query = SearchQuery()
# a query to tally users with failed logins
date_range_match = RangeMatch('utctimestamp', begindateUTC, enddateUTC)
search_query.add_must(date_range_match)
search_query.add_must(PhraseMatch('category', 'authentication'))
search_query.add_must(PhraseMatch('details.success','false'))
search_query.add_must(ExistsMatch('details.username'))
search_query.add_aggregation(Aggregation('details.success'))
search_query.add_aggregation(Aggregation('details.username'))
results = search_query.execute(es_client, indices=['events','events-previous'])
# any usernames or words to ignore
# especially useful if ES is analyzing the username field and breaking apart [email protected]
# into user somewhere and .com
stoplist =self.options.ignoreusernames.split(',')
# walk the aggregate failed users
# and look for successes/failures
for t in results['aggregations']['details.username']['terms']:
if t['key'] in stoplist:
continue
failures = 0
success = 0
username = t['key']
details_query = SearchQuery()
details_query.add_must(date_range_match)
details_query.add_must(PhraseMatch('category', 'authentication'))
details_query.add_must(PhraseMatch('details.username', username))
details_query.add_aggregation(Aggregation('details.success'))
details_results = details_query.execute(es_client)
# details.success is boolean. As an aggregate is an int (0/1)
for details_term in details_results['aggregations']['details.success']['terms']:
if details_term['key'] == 1:
success = details_term['count']
if details_term['key'] == 0:
failures = details_term['count']
resultsList.append(
dict(
username=username,
failures=failures,
success=success,
begin=begindateUTC.isoformat(),
end=enddateUTC.isoformat()
)
)
response.body = json.dumps(resultsList)
response.status = 200
return (request, response)
def num_objects_saved(self):
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(ExistsMatch('keyname'))
results = search_query.execute(self.es_client)
return len(results['hits'])
def test_without_queries(self):
query = SearchQuery(minutes=10)
with pytest.raises(AttributeError):
query.execute(self.es_client)
def test_search_no_results(self):
search_query = SearchQuery()
search_query.add_must(TermMatch('garbagefielddoesntexist', 'testingvalues'))
results = search_query.execute(self.es_client)
assert results['hits'] == []
def test_search_nonexisting_index(self):
search_query = SearchQuery()
search_query.add_must(TermMatch('key', 'value'))
with pytest.raises(ElasticsearchInvalidIndex):
search_query.execute(self.es_client, indices=['doesnotexist'])
# Wait for .kibana index to be ready
num_times = 0
while not client.index_exists(kibana_index_name):
if num_times < 3:
print("Waiting for .kibana index to be ready")
time.sleep(1)
num_times += 1
else:
print(".kibana index not created...exiting")
sys.exit(1)
# Check to see if index patterns exist in .kibana
query = SearchQuery()
query.add_must(TermMatch('_type', 'index-pattern'))
results = query.execute(client, indices=[kibana_index_name])
if len(results['hits']) == 0:
# Create index patterns and assign default index mapping
index_mappings_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'index_mappings')
listing = os.listdir(index_mappings_path)
for infile in listing:
json_file_path = os.path.join(index_mappings_path, infile)
with open(json_file_path) as json_data:
mapping_data = json.load(json_data)
print "Creating {0} index mapping".format(mapping_data['title'])
client.save_object(body=mapping_data, index=kibana_index_name, doc_type='index-pattern', doc_id=mapping_data['title'])
# Assign default index to 'events'
print "Assigning events as default index mapping"
index_name = 'events'
url = '{}/api/kibana/settings/defaultIndex'.format(kibana_url)
