def test_simple_input(self):
self.query.add_must_not(ExistsMatch('note'))
assert self.query.must_not == [ExistsMatch('note')]
def test_populated_array(self):
self.query.add_must(ExistsMatch('details'))
self.query.add_must([ExistsMatch('note'), TermMatch('note', 'test')])
assert self.query.must == [ExistsMatch('details'), ExistsMatch('note'), TermMatch('note', 'test')]
def test_complex_aggregation_query_execute(self):
query = SearchQuery()
assert query.date_timedelta == {}
query.add_must(ExistsMatch('ip'))
query.add_aggregation(Aggregation('ip'))
self.populate_test_event(
{
'summary': 'Test Summary',
'ip': '127.0.0.1',
'details': {
'information': 'Example information'
}
}
)
self.populate_test_event(
{
'summary': 'Test Summary',
'ip': '1.2.3.4',
'details': {
'information': 'Example information'
}
}
)
self.populate_test_event(
{
'summary': 'Test Summary',
'ip': '1.2.3.4',
'details': {
'information': 'Example information'
}
}
)
self.refresh(self.event_index_name)
results = query.execute(self.es_client)
assert results.keys() == ['hits', 'meta', 'aggregations']
assert results['meta'].keys() == ['timed_out']
assert results['meta']['timed_out'] is False
sorted_hits = sorted(results['hits'], key=lambda k: k['_source']['ip'])
assert len(sorted_hits) == 3
assert sorted_hits[0].keys() == ['_score', '_type', '_id', '_source', '_index']
assert type(sorted_hits[0]['_id']) == unicode
assert sorted_hits[0]['_type'] == TMP_DOC_TYPE
assert sorted_hits[0]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert sorted_hits[0]['_source']['ip'] == '1.2.3.4'
assert sorted_hits[0]['_source']['summary'] == 'Test Summary'
assert sorted_hits[1]['_source']['type'] == 'event'
assert sorted_hits[0]['_source']['details'].keys() == ['information']
assert sorted_hits[0]['_source']['details']['information'] == 'Example information'
assert sorted_hits[1].keys() == ['_score', '_type', '_id', '_source', '_index']
assert type(sorted_hits[1]['_id']) == unicode
assert sorted_hits[1]['_type'] == TMP_DOC_TYPE
assert sorted_hits[1]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert sorted_hits[1]['_source']['ip'] == '1.2.3.4'
assert sorted_hits[1]['_source']['summary'] == 'Test Summary'
assert sorted_hits[1]['_source']['type'] == 'event'
assert sorted_hits[1]['_source']['details'].keys() == ['information']
assert sorted_hits[1]['_source']['details']['information'] == 'Example information'
assert type(sorted_hits[2]['_id']) == unicode
assert sorted_hits[2]['_type'] == TMP_DOC_TYPE
assert sorted_hits[2]['_index'] == datetime.now().strftime("events-%Y%m%d")
assert sorted_hits[2]['_source']['ip'] == '127.0.0.1'
assert sorted_hits[2]['_source']['summary'] == 'Test Summary'
assert sorted_hits[2]['_source']['type'] == 'event'
assert sorted_hits[2]['_source']['details'].keys() == ['information']
assert sorted_hits[2]['_source']['details']['information'] == 'Example information'
assert results['aggregations'].keys() == ['ip']
assert results['aggregations']['ip'].keys() == ['terms']
assert len(results['aggregations']['ip']['terms']) == 2
results['aggregations']['ip']['terms'].sort()
assert results['aggregations']['ip']['terms'][0]['count'] == 1
assert results['aggregations']['ip']['terms'][0]['key'] == '127.0.0.1'
assert results['aggregations']['ip']['terms'][1]['count'] == 2
assert results['aggregations']['ip']['terms'][1]['key'] == '1.2.3.4'
def onMessage(self, request, response):
'''
request: http://bottlepy.org/docs/dev/api.html#the-request-object
response: http://bottlepy.org/docs/dev/api.html#the-response-object
'''
# an ES query/facet to count success/failed logins
# oriented to the data having
# category: authentication
# details.success marked true/false for success/failed auth
# details.username as the user
begindateUTC = None
enddateUTC = None
resultsList = list()
if begindateUTC is None:
begindateUTC = datetime.now() - timedelta(hours=12)
begindateUTC = toUTC(begindateUTC)
if enddateUTC is None:
enddateUTC = datetime.now()
enddateUTC = toUTC(enddateUTC)
es_client = ElasticsearchClient(
list('{0}'.format(s) for s in self.restoptions['esservers']))
search_query = SearchQuery()
# a query to tally users with failed logins
date_range_match = RangeMatch('utctimestamp', begindateUTC, enddateUTC)
search_query.add_must(date_range_match)
search_query.add_must(PhraseMatch('category', 'authentication'))
search_query.add_must(PhraseMatch('details.success', 'false'))
search_query.add_must(ExistsMatch('details.username'))
search_query.add_aggregation(Aggregation('details.success'))
search_query.add_aggregation(Aggregation('details.username'))
results = search_query.execute(es_client,
indices=['events', 'events-previous'])
# any usernames or words to ignore
# especially useful if ES is analyzing the username field and breaking apart [email protected]
# into user somewhere and .com
stoplist = self.options.ignoreusernames.split(',')
# walk the aggregate failed users
# and look for successes/failures
for t in results['aggregations']['details.username']['terms']:
if t['key'] in stoplist:
continue
failures = 0
success = 0
username = t['key']
details_query = SearchQuery()
details_query.add_must(date_range_match)
details_query.add_must(PhraseMatch('category', 'authentication'))
details_query.add_must(PhraseMatch('details.username', username))
details_query.add_aggregation(Aggregation('details.success'))
details_results = details_query.execute(es_client)
# details.success is boolean. As an aggregate is an int (0/1)
for details_term in details_results['aggregations'][
'details.success']['terms']:
if details_term['key'] == 1:
success = details_term['count']
if details_term['key'] == 0:
failures = details_term['count']
resultsList.append(
dict(username=username,
failures=failures,
success=success,
begin=begindateUTC.isoformat(),
end=enddateUTC.isoformat()))
response.body = json.dumps(resultsList)
response.status = 200
return (request, response)
def test_simple_input(self):
self.query.add_should(ExistsMatch('note'))
assert self.query.should == [ExistsMatch('note')]
def num_objects_saved(self):
self.refresh(self.event_index_name)
search_query = SearchQuery()
search_query.add_must(ExistsMatch('keyname'))
results = search_query.execute(self.es_client)
return len(results['hits'])
