def setUp(self): self.body_build = SearchBodyBuild() self.body_build.range('time', 'start', 'end')\ .must('ip', '10.10.0.0')\ .groupby_date('time', '1d')\ .groupby('domain')\ .sum('pv') self.res = { 'hits': {}, '_shards': {}, 'took': 100, 'aggregations': { 'time-date_histogram': { 'buckets': [ { 'domain-terms': { 'buckets': [ { 'pv-sum': { 'value': 2000 }, 'key': 'www.xxxx.com', 'doc_count': 1000 } ], 'sum_other_doc_count': 0, 'doc_count_error_upper_bound': 0 }, 'key_as_string': '1111-11-11', 'key': 11111111, 'doc_count': 11111111 }, { 'domain-terms': { 'buckets': [ { 'pv-sum': { 'value': 2000 }, 'key': 'www.xxxx.com', 'doc_count': 1000 } ], 'sum_other_doc_count': 0, 'doc_count_error_upper_bound': 0 }, 'key_as_string': '2222-22-22', 'key': 22222222, 'doc_count': 22222222 } ], } } } self.pre_parser = parse_aggs.parse(self.body_build, self.res)
def setUp(self): self.esbody = SearchBodyBuild() self.testquery = { 'query': { 'bool': { 'must': [ { 'bool': { 'should': [] } }, ] } } } self.testaggs = {}
class Test_SearchBodyBuild(unittest.TestCase): def setUp(self): self.esbody = SearchBodyBuild() self.testquery = { 'query': { 'bool': { 'must': [ { 'bool': { 'should': [] } }, ] } } } self.testaggs = {} def test_select_must(self): self.esbody.must('time1', 'today') self.testquery['query']['bool']['must'].append( {'term': { 'time1': 'today' }}) self.assertEqual(self.esbody.query_build.get_body(), self.testquery) self.esbody.must('time2', 'today') self.testquery['query']['bool']['must'].append( {'term': { 'time2': 'today' }}) self.assertEqual(self.esbody.query_build.get_body(), self.testquery) def test_select_should(self): self.esbody.should('time1', 'today')\ .should('time2', 'today') self.testquery['query']['bool']['must'][0]['bool']['should'].append( {'term': { 'time1': 'today' }}) self.testquery['query']['bool']['must'][0]['bool']['should'].append( {'term': { 'time2': 'today' }}) self.assertEqual(self.esbody.query_build.get_body(), self.testquery) def test_select_range(self): self.esbody.range('time', 'yesterday', 'today') self.testquery['query']['bool']['must'].append( {'range': { 'time': { 'gte': 'yesterday', 'lte': 'today' } }}) self.assertEqual(self.esbody.query_build.get_body(), self.testquery) def test_select_range_should(self): self.esbody.range('time', 'yesterday', 'today')\ .should('time1', 'today')\ .should('time2', 'today') self.testquery['query']['bool']['must'].append( {'range': { 'time': { 'gte': 'yesterday', 'lte': 'today' } }}) self.testquery['query']['bool']['must'][0]['bool']['should'].append( {'term': { 'time1': 'today' }}) self.testquery['query']['bool']['must'][0]['bool']['should'].append( {'term': { 'time2': 'today' }}) self.assertEqual(self.esbody.query_build.get_body(), self.testquery) def test_update_aggs(self): self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs) self.esbody.groupby('ip') self.testaggs['aggs'] = { 'ip-terms': { 'terms': { 'field': 'ip', 'size': 0 } } } self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs) self.esbody.groupby_date('date', '1h')\ .avg('count') self.testaggs['aggs']['ip-terms']['aggs'] = { 'date-date_histogram': { 'date_histogram': { 'field': 'date', 'interval': '1h' } }, 'count-avg': { 'avg': { 'field': 'count' } } } self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs) def test_bodybuild(self): body = {} body.update(self.testquery) body.update(self.testaggs) self.assertEqual(self.esbody.get_body(), body) self.esbody.range('time', 'yesterday', 'today')\ .should('time1', 'today') self.testquery['query']['bool']['must'].append( {'range': { 'time': { 'gte': 'yesterday', 'lte': 'today' } }}) self.testquery['query']['bool']['must'][0]['bool']['should'].append( {'term': { 'time1': 'today' }}) self.esbody.groupby_date('date', '1h')\ .avg('count') self.testaggs['aggs'] = { 'date-date_histogram': { 'date_histogram': { 'field': 'date', 'interval': '1h' } }, 'count-avg': { 'avg': { 'field': 'count' } } } body = {} body.update(self.testquery) body.update(self.testaggs) self.assertEqual(self.esbody.get_body(), body)
class Test_SearchBodyBuild(unittest.TestCase): def setUp(self): self.esbody = SearchBodyBuild() self.testquery = { 'query': { 'bool': { 'must': [ { 'bool': { 'should': [] } }, ] } } } self.testaggs = {} def test_select_must(self): self.esbody.must('time1', 'today') self.testquery['query']['bool']['must'].append({'term': {'time1': 'today'}}) self.assertEqual( self.esbody.query_build.get_body(), self.testquery ) self.esbody.must('time2', 'today') self.testquery['query']['bool']['must'].append({'term': {'time2': 'today'}}) self.assertEqual( self.esbody.query_build.get_body(), self.testquery ) def test_select_should(self): self.esbody.should('time1', 'today')\ .should('time2', 'today') self.testquery['query']['bool']['must'][0]['bool']['should'].append({'term': {'time1': 'today'}}) self.testquery['query']['bool']['must'][0]['bool']['should'].append({'term': {'time2': 'today'}}) self.assertEqual( self.esbody.query_build.get_body(), self.testquery ) def test_select_range(self): self.esbody.range('time', 'yesterday', 'today') self.testquery['query']['bool']['must'].append({'range': {'time': {'gte': 'yesterday', 'lte': 'today'}}}) self.assertEqual( self.esbody.query_build.get_body(), self.testquery ) def test_select_range_should(self): self.esbody.range('time', 'yesterday', 'today')\ .should('time1', 'today')\ .should('time2', 'today') self.testquery['query']['bool']['must'].append({'range': {'time': {'gte': 'yesterday', 'lte': 'today'}}}) self.testquery['query']['bool']['must'][0]['bool']['should'].append({'term': {'time1': 'today'}}) self.testquery['query']['bool']['must'][0]['bool']['should'].append({'term': {'time2': 'today'}}) self.assertEqual( self.esbody.query_build.get_body(), self.testquery ) def test_update_aggs(self): self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs) self.esbody.groupby('ip') self.testaggs['aggs'] = {'ip-terms': {'terms': {'field': 'ip', 'size': 0}}} self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs) self.esbody.groupby_date('date', '1h')\ .avg('count') self.testaggs['aggs']['ip-terms']['aggs'] = { 'date-date_histogram': {'date_histogram': {'field': 'date', 'interval': '1h'}}, 'count-avg': {'avg': {'field': 'count'}} } self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs) def test_bodybuild(self): body = {} body.update(self.testquery) body.update(self.testaggs) self.assertEqual(self.esbody.get_body(), body) self.esbody.range('time', 'yesterday', 'today')\ .should('time1', 'today') self.testquery['query']['bool']['must'].append({'range': {'time': {'gte': 'yesterday', 'lte': 'today'}}}) self.testquery['query']['bool']['must'][0]['bool']['should'].append({'term': {'time1': 'today'}}) self.esbody.groupby_date('date', '1h')\ .avg('count') self.testaggs['aggs'] = { 'date-date_histogram': {'date_histogram': {'field': 'date', 'interval': '1h'}}, 'count-avg': {'avg': {'field': 'count'}} } body = {} body.update(self.testquery) body.update(self.testaggs) self.assertEqual(self.esbody.get_body(), body)
import datetime from esearch.query import SearchBodyBuild from esearch import parse_aggs # Construct the search body body_build = SearchBodyBuild() end = datetime.datetime.now() start = end - datetime.timedelta(days=10) body = body_build.range('time', start, end)\ .must('ip', '10.10.0.0')\ .groupby_date('time', '1d')\ .groupby('domain')\ .sum('pv')\ .get_body() print body # Search # import elasticsearch # es_client = elasticsearch.Elasticsearch() # res = es_client.search(index='test', body=body, doc_type='test') res = { 'hits': {}, '_shards': {}, 'took': 100, 'aggregations': { 'time-date_histogram': { 'buckets': [{ 'domain-terms': { 'buckets': [{
class Test_ParseAggs(unittest.TestCase): def setUp(self): self.body_build = SearchBodyBuild() self.body_build.range('time', 'start', 'end')\ .must('ip', '10.10.0.0')\ .groupby_date('time', '1d')\ .groupby('domain')\ .sum('pv') self.res = { 'hits': {}, '_shards': {}, 'took': 100, 'aggregations': { 'time-date_histogram': { 'buckets': [ { 'domain-terms': { 'buckets': [ { 'pv-sum': { 'value': 2000 }, 'key': 'www.xxxx.com', 'doc_count': 1000 } ], 'sum_other_doc_count': 0, 'doc_count_error_upper_bound': 0 }, 'key_as_string': '1111-11-11', 'key': 11111111, 'doc_count': 11111111 }, { 'domain-terms': { 'buckets': [ { 'pv-sum': { 'value': 2000 }, 'key': 'www.xxxx.com', 'doc_count': 1000 } ], 'sum_other_doc_count': 0, 'doc_count_error_upper_bound': 0 }, 'key_as_string': '2222-22-22', 'key': 22222222, 'doc_count': 22222222 } ], } } } self.pre_parser = parse_aggs.parse(self.body_build, self.res) def get_date_and_value(self, idx): idx = str(idx) return '%s-%s-%s' % (idx*4, idx*2, idx*2), int(idx*8) def test_get_buckets_metric(self): for idx, time_bucket in enumerate(self.pre_parser.get_buckets('time')): date, value = self.get_date_and_value(idx+1) self.assertEqual(time_bucket.get_key_as_string(), date) self.assertEqual(time_bucket.get_key(), value) self.assertEqual(time_bucket.get_doc_count(), value) for domain_bucket in time_bucket.get_buckets('domain'): self.assertEqual(domain_bucket.get_key(), 'www.xxxx.com') self.assertEqual(domain_bucket.get_doc_count(), 1000) self.assertEqual(domain_bucket.get_sum('pv'), 2000) with self.assertRaises(Exception): domain_bucket.get_buckets('xxxxx')
import datetime from esearch.query import SearchBodyBuild from esearch import parse_aggs # Construct the search body body_build = SearchBodyBuild() end = datetime.datetime.now() start = end - datetime.timedelta(days=10) body = body_build.range('time', start, end)\ .must('ip', '10.10.0.0')\ .groupby_date('time', '1d')\ .groupby('domain')\ .sum('pv')\ .get_body() print body # Search # import elasticsearch # es_client = elasticsearch.Elasticsearch() # res = es_client.search(index='test', body=body, doc_type='test') res = { 'hits': {}, '_shards': {}, 'took': 100, 'aggregations': { 'time-date_histogram': { 'buckets': [