def query_tests(self):
tests = {
ExistsMatch('summary'): [{
'note': 'example note'
}, {
'sum': 'example sum'
}, {
'details': {
'note': 'example note'
},
}],
ExistsMatch('details.note'): [
{
'summary': 'garbage summary',
'details': {
'ipaddress': 'test'
}
},
# {
# 'details': 'no details', # Currently, this throws MapperParsingException
# todo: fix code to handle this elegantly
# },
]
}
return tests
def query_tests(self):
tests = [[
ExistsMatch('summary'),
[
{
'note': 'example note'
},
{
'sum': 'example sum'
},
{
'details': {
'note': 'example note'
}
},
]
],
[
ExistsMatch('details.note'),
[
{
'summary': 'garbage summary',
'details': {
'ipaddress': 'test'
}
},
]
]]
return tests
def test_execute_with_should(self):
self.populate_example_event()
self.refresh(self.event_index_name)
self.query.add_should(ExistsMatch('summary'))
self.query.add_should(ExistsMatch('nonexistentfield'))
results = self.query.execute(self.es_client)
assert len(results['hits']) == 1
def test_populated_array(self):
self.query.add_should(ExistsMatch('details'))
self.query.add_should([ExistsMatch('note'), TermMatch('note', 'test')])
assert self.query.should == [
ExistsMatch('details'),
ExistsMatch('note'),
TermMatch('note', 'test')
]
def test_populated_array(self):
self.query.add_must_not(ExistsMatch('details'))
self.query.add_must_not(
[ExistsMatch('note'),
TermMatch('note', 'test')])
assert self.query.must_not == [
ExistsMatch('details'),
ExistsMatch('note'),
TermMatch('note', 'test')
]
def main(self):
self.parse_config('duo_authfail.conf', ['url'])
search_query = SearchQuery(minutes=15)
search_query.add_must([
TermMatch('category', 'authentication'),
ExistsMatch('details.sourceipaddress'),
ExistsMatch('details.username'),
PhraseMatch('details.result', 'FRAUD')
])
self.filtersManual(search_query)
self.searchEventsSimple()
self.walkEvents()
def test_array_input(self):
queries = [
ExistsMatch('note'),
TermMatch('note', 'test')
]
self.query.add_should(queries)
assert self.query.should == queries
def test_array_input(self):
queries = [
ExistsMatch('note'),
TermMatch('note', 'test')
]
self.query.add_must_not(queries)
assert self.query.must_not == queries
def test_simple_query_execute(self):
query = SearchQuery()
query.add_must(ExistsMatch('note'))
assert query.date_timedelta == {}
self.populate_example_event()
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert sorted(results.keys()) == ['hits', 'meta']
assert list(results['meta'].keys()) == ['timed_out']
assert results['meta']['timed_out'] is False
assert len(results['hits']) == 1
assert sorted(results['hits'][0].keys()) == ['_id', '_index', '_score', '_source']
assert type(results['hits'][0]['_id']) == str
assert results['hits'][0]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert results['hits'][0]['_source']['note'] == 'Example note'
assert results['hits'][0]['_source']['summary'] == 'Test Summary'
assert results['hits'][0]['_source']['type'] == 'event'
assert list(results['hits'][0]['_source']['details'].keys()) == ['information']
assert results['hits'][0]['_source']['details']['information'] == 'Example information'
with pytest.raises(KeyError):
results['abcdefg']
with pytest.raises(KeyError):
results['abcdefg']['test']
def getESAlerts(es):
search_query = SearchQuery(minutes=50)
# We use an ExistsMatch here just to satisfy the
# requirements of a search query must have some "Matchers"
search_query.add_must(ExistsMatch('summary'))
results = search_query.execute(es, indices=['alerts'], size=10000)
return results
def test_writing_with_type(self):
query = SearchQuery()
default_event = {
"_type": "example",
"_source": {
"receivedtimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
}
self.populate_test_event(default_event)
self.refresh(self.event_index_name)
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
assert len(results['hits']) == 1
assert sorted(results['hits'][0].keys()) == [
'_id', '_index', '_score', '_source', '_type'
]
assert results['hits'][0]['_type'] == 'example'
assert results['hits'][0]['_source']['summary'] == 'Test summary'
assert results['hits'][0]['_source']['details'] == {
"note": "Example note"
}
def test_without_time_defined(self):
query = SearchQuery()
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {}
default_event = {
"utctimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(default_event)
default_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'days': 11})
default_event[
'receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'days': 11})
self.populate_test_event(default_event)
not_old_event = default_event
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'days': 9})
not_old_event[
'receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'days': 9})
self.populate_test_event(not_old_event)
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 3
def test_beginning_time_seconds_received_timestamp(self):
query = SearchQuery(seconds=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'seconds': 10}
default_event = {
"receivedtimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(default_event)
too_old_event = default_event
too_old_event[
'receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'seconds': 11})
too_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'seconds': 11})
self.populate_test_event(too_old_event)
not_old_event = default_event
not_old_event[
'receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'seconds': 9})
not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp(
{'seconds': 9})
self.populate_test_event(not_old_event)
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 2
def test_writing_event_defaults(self):
query = SearchQuery()
default_event = {}
self.populate_test_event(default_event)
self.refresh(self.event_index_name)
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
assert len(results['hits']) == 1
assert sorted(results['hits'][0].keys()) == [
'_id', '_index', '_score', '_source', '_type'
]
saved_event = results['hits'][0]['_source']
assert 'category' in saved_event
assert 'details' in saved_event
assert 'hostname' in saved_event
assert 'mozdefhostname' in saved_event
assert 'processid' in saved_event
assert 'processname' in saved_event
assert 'receivedtimestamp' in saved_event
assert 'severity' in saved_event
assert 'source' in saved_event
assert 'summary' in saved_event
assert 'tags' in saved_event
assert 'timestamp' in saved_event
assert 'utctimestamp' in saved_event
assert 'category' in saved_event
def main(self):
self.parse_config('triagebot_escalation.conf', ['url', 'severity'])
search_query = SearchQuery(minutes=15)
search_query.add_must([
TermMatch('category', 'triagebot'),
TermMatch('details.userresponse', 'no'),
ExistsMatch('details.email'),
ExistsMatch('details.identifier')
])
self.filtersManual(search_query)
# Search events
self.searchEventsSimple()
self.walkEvents()
def test_execute_without_size(self):
for num in range(0, 1200):
self.populate_example_event()
self.refresh(self.event_index_name)
query = SearchQuery()
query.add_must(ExistsMatch('summary'))
results = query.execute(self.es_client)
assert len(results['hits']) == 1000
def search_and_verify_event(self, expected_event):
self.refresh('events')
search_query = SearchQuery(minutes=5)
search_query.add_must(ExistsMatch('tags'))
results = search_query.execute(self.es_client)
assert len(results['hits']) == 1
saved_event = results['hits'][0]['_source']
self.verify_event(saved_event, expected_event)
def searchEventsAggregated(self, aggregationPath, samplesLimit=5):
"""
Search events, aggregate matching ES filters by aggregationPath,
store them in self.aggregations as a list of dictionaries
keys:
value: the text value that was found in the aggregationPath
count: the hitcount of the text value
events: the sampled list of events that matched
allevents: the unsample, total list of matching events
aggregationPath can be key.subkey.subkey to specify a path to a dictionary value
relative to the _source that's returned from elastic search.
ex: details.sourceipaddress
"""
# We automatically add the key that we're matching on
# for aggregation, as a query requirement
aggreg_key_exists = ExistsMatch(aggregationPath)
if aggreg_key_exists not in self.main_query.must:
self.main_query.add_must(aggreg_key_exists)
try:
esresults = self.main_query.execute(self.es,
indices=self.event_indices)
results = esresults['hits']
# List of aggregation values that can be counted/summarized by Counter
# Example: ['*****@*****.**','*****@*****.**', '*****@*****.**'] for an email aggregField
aggregationValues = []
for r in results:
aggregationValues.append(
getValueByPath(r['_source'], aggregationPath))
# [{value:'*****@*****.**',count:1337,events:[...]}, ...]
aggregationList = []
for i in Counter(aggregationValues).most_common():
idict = {
'value': i[0],
'count': i[1],
'events': [],
'allevents': []
}
for r in results:
if getValueByPath(r['_source'], aggregationPath).encode(
'ascii', 'ignore') == i[0]:
# copy events detail into this aggregation up to our samples limit
if len(idict['events']) < samplesLimit:
idict['events'].append(r)
# also copy all events to a non-sampled list
# so we mark all events as alerted and don't re-alert
idict['allevents'].append(r)
aggregationList.append(idict)
self.aggregations = aggregationList
self.log.debug(self.aggregations)
except Exception as e:
self.log.error('Error while searching events in ES: {0}'.format(e))
def test_aggregation_with_aggregation_size(self):
for num in range(0, 100):
event = {'keyname': 'value' + str(num)}
self.populate_test_object(event)
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(ExistsMatch('keyname'))
search_query.add_aggregation(Aggregation('keyname', 2))
results = search_query.execute(self.es_client)
assert len(results['aggregations']['keyname']['terms']) == 2
def query_tests(self):
tests = {
ExistsMatch('summary'): [
{
'summary': 'test'
},
{
'summary': 'example test summary'
},
],
ExistsMatch('details.note'): [
{
'summary': 'garbage summary',
'details': {
'note': 'test'
}
},
]
}
return tests
def test_aggregation_query_execute(self):
query = SearchQuery()
query.add_must(ExistsMatch('note'))
query.add_aggregation(Aggregation('note'))
assert query.date_timedelta == {}
self.populate_example_event()
self.populate_example_event()
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert results.keys() == ['hits', 'meta', 'aggregations']
assert results['meta'].keys() == ['timed_out']
assert results['meta']['timed_out'] is False
assert len(results['hits']) == 2
assert results['hits'][0].keys() == ['_score', '_type', '_id', '_source', '_index']
assert type(results['hits'][0]['_id']) == unicode
assert results['hits'][0]['_type'] == TMP_DOC_TYPE
assert results['hits'][0]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert results['hits'][0]['_source']['note'] == 'Example note'
assert results['hits'][0]['_source']['summary'] == 'Test Summary'
assert results['hits'][0]['_source']['type'] == 'event'
assert results['hits'][0]['_source']['details'].keys() == ['information']
assert results['hits'][0]['_source']['details']['information'] == 'Example information'
assert results['hits'][1].keys() == ['_score', '_type', '_id', '_source', '_index']
assert type(results['hits'][1]['_id']) == unicode
assert results['hits'][1]['_type'] == TMP_DOC_TYPE
assert results['hits'][1]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert results['hits'][1]['_source']['note'] == 'Example note'
assert results['hits'][1]['_source']['summary'] == 'Test Summary'
assert results['hits'][1]['_source']['type'] == 'event'
assert results['hits'][1]['_source']['details'].keys() == ['information']
assert results['hits'][1]['_source']['details']['information'] == 'Example information'
assert results['aggregations'].keys() == ['note']
assert results['aggregations']['note'].keys() == ['terms']
assert len(results['aggregations']['note']['terms']) == 1
results['aggregations']['note']['terms'].sort()
assert results['aggregations']['note']['terms'][0]['count'] == 2
assert results['aggregations']['note']['terms'][0]['key'] == 'Example note'
def main(self):
search_query = SearchQuery(minutes=20)
search_query.add_must([
TermMatch('source', 'cloudtrail'),
TermMatch('details.eventname', 'PutBucketPolicy'),
ExistsMatch(
'details.requestparameters.bucketpolicy.statement.principal')
])
self.filtersManual(search_query)
self.searchEventsSimple()
self.walkEvents()
def main(self):
search_query = SearchQuery(minutes=15)
search_query.add_must([
TermMatch('eventsource', 'nsm'),
TermMatch('category', 'brointel'),
TermMatch('details.sources', 'abuse.ch SSLBL'),
ExistsMatch('details.sourceipaddress')
])
self.filtersManual(search_query)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
search_query = SearchQuery(minutes=2)
search_query.add_must([
TermMatch('category', 'bro'),
TermMatch('source', 'intel'),
ExistsMatch('details.seenindicator'),
TermsMatch('hostname', ['sensor1', 'sensor2', 'sensor3'])
])
self.filtersManual(search_query)
# Search aggregations on field 'seenindicator', keep X samples of events at most
self.searchEventsAggregated('details.seenindicator', samplesLimit=10)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=10)
def main(self):
# Create a query to look back the last 20 minutes
search_query = SearchQuery(minutes=5)
# Add search terms to our query
search_query.add_must([
TermMatch('source', 'cloudtrail'),
TermMatch('details.eventverb', 'Describe'),
ExistsMatch('details.sourceipv4address')
])
self.filtersManual(search_query)
# We aggregate on details.sourceipv4address which is the AWS service name
self.searchEventsAggregated('details.sourceipv4address',
samplesLimit=2)
self.walkAggregations(threshold=5)
def main(self):
# Create a query to look back the last 20 minutes
search_query = SearchQuery(minutes=20)
# Add search terms to our query
search_query.add_must([
TermMatch('source', 'guardduty'),
TermMatch('details.finding.action.actionType', 'PORT_PROBE'),
ExistsMatch('details.sourceipaddress'),
])
self.filtersManual(search_query)
# Search aggregations on field 'sourceipaddress'
# keep X samples of events at most
self.searchEventsAggregated('details.sourceipaddress', samplesLimit=10)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=1)
def test_time_received_timestamp(self):
query = SearchQuery(seconds=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'seconds': 10}
received_timestamp_default_event = {
"receivedtimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(received_timestamp_default_event)
utctimestamp_default_event = {
"utctimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(utctimestamp_default_event)
default_event = {
"utctimestamp": UnitTestSuite.current_timestamp(),
"receivedtimestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_event(default_event)
modified_received_timestamp_event = default_event
modified_received_timestamp_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 11})
self.populate_test_event(modified_received_timestamp_event)
modified_utc_timestamp_event = default_event
modified_utc_timestamp_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 9})
self.populate_test_event(modified_utc_timestamp_event)
self.flush(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 5
def test_without_utctimestamp(self):
query = SearchQuery(days=10)
query.add_must(ExistsMatch('summary'))
assert query.date_timedelta == {'days': 10}
default_event = {
"timestamp": UnitTestSuite.current_timestamp(),
"summary": "Test summary",
"details": {
"note": "Example note",
}
}
self.populate_test_object(default_event)
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert len(results['hits']) == 0
def test_simple_writing_event_string(self):
event = json.dumps({"key": "example value for string of json test"})
self.es_client.save_event(body=event)
self.refresh(self.event_index_name)
num_events = self.get_num_events()
assert num_events == 1
query = SearchQuery()
query.add_must(ExistsMatch('key'))
results = query.execute(self.es_client)
assert sorted(results['hits'][0].keys()) == [
'_id', '_index', '_score', '_source', '_type'
]
assert results['hits'][0]['_source'][
'key'] == 'example value for string of json test'
assert len(results['hits']) == 1
assert results['hits'][0]['_type'] == 'event'
def verify_events(options):
es_client = ElasticsearchClient(options.esservers)
for required_field in options.required_fields:
logger.debug('Looking for events without ' + required_field)
search_query = SearchQuery(hours=12)
search_query.add_must_not(ExistsMatch(required_field))
# Exclude all events that are mozdef related health and stats
search_query.add_must_not(TermMatch('_type', 'mozdefstats'))
search_query.add_must_not(TermMatch('_type', 'mozdefhealth'))
search_query.add_aggregation(Aggregation('_type'))
# We don't care about the actual events, we only want the numbers
results = search_query.execute(es_client, size=1)
for aggreg_term in results['aggregations']['_type']['terms']:
count = aggreg_term['count']
category = aggreg_term['key']
logger.error(
"Found {0} bad events of _type '{1}' missing '{2}' field".
format(count, category, required_field))