def test_simple_input(self): self.query.add_must_not(ExistsMatch('note')) assert self.query.must_not == [ExistsMatch('note')]
def test_populated_array(self): self.query.add_must(ExistsMatch('details')) self.query.add_must([ExistsMatch('note'), TermMatch('note', 'test')]) assert self.query.must == [ExistsMatch('details'), ExistsMatch('note'), TermMatch('note', 'test')]
def test_complex_aggregation_query_execute(self): query = SearchQuery() assert query.date_timedelta == {} query.add_must(ExistsMatch('ip')) query.add_aggregation(Aggregation('ip')) self.populate_test_event( { 'summary': 'Test Summary', 'ip': '127.0.0.1', 'details': { 'information': 'Example information' } } ) self.populate_test_event( { 'summary': 'Test Summary', 'ip': '1.2.3.4', 'details': { 'information': 'Example information' } } ) self.populate_test_event( { 'summary': 'Test Summary', 'ip': '1.2.3.4', 'details': { 'information': 'Example information' } } ) self.refresh(self.event_index_name) results = query.execute(self.es_client) assert results.keys() == ['hits', 'meta', 'aggregations'] assert results['meta'].keys() == ['timed_out'] assert results['meta']['timed_out'] is False sorted_hits = sorted(results['hits'], key=lambda k: k['_source']['ip']) assert len(sorted_hits) == 3 assert sorted_hits[0].keys() == ['_score', '_type', '_id', '_source', '_index'] assert type(sorted_hits[0]['_id']) == unicode assert sorted_hits[0]['_type'] == TMP_DOC_TYPE assert sorted_hits[0]['_index'] == datetime.now().strftime("events-%Y%m%d") assert sorted_hits[0]['_source']['ip'] == '1.2.3.4' assert sorted_hits[0]['_source']['summary'] == 'Test Summary' assert sorted_hits[1]['_source']['type'] == 'event' assert sorted_hits[0]['_source']['details'].keys() == ['information'] assert sorted_hits[0]['_source']['details']['information'] == 'Example information' assert sorted_hits[1].keys() == ['_score', '_type', '_id', '_source', '_index'] assert type(sorted_hits[1]['_id']) == unicode assert sorted_hits[1]['_type'] == TMP_DOC_TYPE assert sorted_hits[1]['_index'] == datetime.now().strftime("events-%Y%m%d") assert sorted_hits[1]['_source']['ip'] == '1.2.3.4' assert sorted_hits[1]['_source']['summary'] == 'Test Summary' assert sorted_hits[1]['_source']['type'] == 'event' assert sorted_hits[1]['_source']['details'].keys() == ['information'] assert sorted_hits[1]['_source']['details']['information'] == 'Example information' assert type(sorted_hits[2]['_id']) == unicode assert sorted_hits[2]['_type'] == TMP_DOC_TYPE assert sorted_hits[2]['_index'] == datetime.now().strftime("events-%Y%m%d") assert sorted_hits[2]['_source']['ip'] == '127.0.0.1' assert sorted_hits[2]['_source']['summary'] == 'Test Summary' assert sorted_hits[2]['_source']['type'] == 'event' assert sorted_hits[2]['_source']['details'].keys() == ['information'] assert sorted_hits[2]['_source']['details']['information'] == 'Example information' assert results['aggregations'].keys() == ['ip'] assert results['aggregations']['ip'].keys() == ['terms'] assert len(results['aggregations']['ip']['terms']) == 2 results['aggregations']['ip']['terms'].sort() assert results['aggregations']['ip']['terms'][0]['count'] == 1 assert results['aggregations']['ip']['terms'][0]['key'] == '127.0.0.1' assert results['aggregations']['ip']['terms'][1]['count'] == 2 assert results['aggregations']['ip']['terms'][1]['key'] == '1.2.3.4'
def onMessage(self, request, response): ''' request: http://bottlepy.org/docs/dev/api.html#the-request-object response: http://bottlepy.org/docs/dev/api.html#the-response-object ''' # an ES query/facet to count success/failed logins # oriented to the data having # category: authentication # details.success marked true/false for success/failed auth # details.username as the user begindateUTC = None enddateUTC = None resultsList = list() if begindateUTC is None: begindateUTC = datetime.now() - timedelta(hours=12) begindateUTC = toUTC(begindateUTC) if enddateUTC is None: enddateUTC = datetime.now() enddateUTC = toUTC(enddateUTC) es_client = ElasticsearchClient( list('{0}'.format(s) for s in self.restoptions['esservers'])) search_query = SearchQuery() # a query to tally users with failed logins date_range_match = RangeMatch('utctimestamp', begindateUTC, enddateUTC) search_query.add_must(date_range_match) search_query.add_must(PhraseMatch('category', 'authentication')) search_query.add_must(PhraseMatch('details.success', 'false')) search_query.add_must(ExistsMatch('details.username')) search_query.add_aggregation(Aggregation('details.success')) search_query.add_aggregation(Aggregation('details.username')) results = search_query.execute(es_client, indices=['events', 'events-previous']) # any usernames or words to ignore # especially useful if ES is analyzing the username field and breaking apart [email protected] # into user somewhere and .com stoplist = self.options.ignoreusernames.split(',') # walk the aggregate failed users # and look for successes/failures for t in results['aggregations']['details.username']['terms']: if t['key'] in stoplist: continue failures = 0 success = 0 username = t['key'] details_query = SearchQuery() details_query.add_must(date_range_match) details_query.add_must(PhraseMatch('category', 'authentication')) details_query.add_must(PhraseMatch('details.username', username)) details_query.add_aggregation(Aggregation('details.success')) details_results = details_query.execute(es_client) # details.success is boolean. As an aggregate is an int (0/1) for details_term in details_results['aggregations'][ 'details.success']['terms']: if details_term['key'] == 1: success = details_term['count'] if details_term['key'] == 0: failures = details_term['count'] resultsList.append( dict(username=username, failures=failures, success=success, begin=begindateUTC.isoformat(), end=enddateUTC.isoformat())) response.body = json.dumps(resultsList) response.status = 200 return (request, response)
def test_simple_input(self): self.query.add_should(ExistsMatch('note')) assert self.query.should == [ExistsMatch('note')]
def num_objects_saved(self): self.refresh(self.event_index_name) search_query = SearchQuery() search_query.add_must(ExistsMatch('keyname')) results = search_query.execute(self.es_client) return len(results['hits'])