def setUp(self):
self.body_build = SearchBodyBuild()
self.body_build.range('time', 'start', 'end')\
.must('ip', '10.10.0.0')\
.groupby_date('time', '1d')\
.groupby('domain')\
.sum('pv')
self.res = {
'hits': {},
'_shards': {},
'took': 100,
'aggregations': {
'time-date_histogram': {
'buckets': [
{
'domain-terms': {
'buckets': [
{
'pv-sum': {
'value': 2000
},
'key': 'www.xxxx.com',
'doc_count': 1000
}
],
'sum_other_doc_count': 0,
'doc_count_error_upper_bound': 0
},
'key_as_string': '1111-11-11',
'key': 11111111,
'doc_count': 11111111
},
{
'domain-terms': {
'buckets': [
{
'pv-sum': {
'value': 2000
},
'key': 'www.xxxx.com',
'doc_count': 1000
}
],
'sum_other_doc_count': 0,
'doc_count_error_upper_bound': 0
},
'key_as_string': '2222-22-22',
'key': 22222222,
'doc_count': 22222222
}
],
}
}
}
self.pre_parser = parse_aggs.parse(self.body_build, self.res)
def setUp(self):
self.esbody = SearchBodyBuild()
self.testquery = {
'query': {
'bool': {
'must': [
{
'bool': {
'should': []
}
},
]
}
}
}
self.testaggs = {}
def setUp(self):
self.esbody = SearchBodyBuild()
self.testquery = {
'query': {
'bool': {
'must': [
{
'bool': {
'should': []
}
},
]
}
}
}
self.testaggs = {}
class Test_SearchBodyBuild(unittest.TestCase):
def setUp(self):
self.esbody = SearchBodyBuild()
self.testquery = {
'query': {
'bool': {
'must': [
{
'bool': {
'should': []
}
},
]
}
}
}
self.testaggs = {}
def test_select_must(self):
self.esbody.must('time1', 'today')
self.testquery['query']['bool']['must'].append(
{'term': {
'time1': 'today'
}})
self.assertEqual(self.esbody.query_build.get_body(), self.testquery)
self.esbody.must('time2', 'today')
self.testquery['query']['bool']['must'].append(
{'term': {
'time2': 'today'
}})
self.assertEqual(self.esbody.query_build.get_body(), self.testquery)
def test_select_should(self):
self.esbody.should('time1', 'today')\
.should('time2', 'today')
self.testquery['query']['bool']['must'][0]['bool']['should'].append(
{'term': {
'time1': 'today'
}})
self.testquery['query']['bool']['must'][0]['bool']['should'].append(
{'term': {
'time2': 'today'
}})
self.assertEqual(self.esbody.query_build.get_body(), self.testquery)
def test_select_range(self):
self.esbody.range('time', 'yesterday', 'today')
self.testquery['query']['bool']['must'].append(
{'range': {
'time': {
'gte': 'yesterday',
'lte': 'today'
}
}})
self.assertEqual(self.esbody.query_build.get_body(), self.testquery)
def test_select_range_should(self):
self.esbody.range('time', 'yesterday', 'today')\
.should('time1', 'today')\
.should('time2', 'today')
self.testquery['query']['bool']['must'].append(
{'range': {
'time': {
'gte': 'yesterday',
'lte': 'today'
}
}})
self.testquery['query']['bool']['must'][0]['bool']['should'].append(
{'term': {
'time1': 'today'
}})
self.testquery['query']['bool']['must'][0]['bool']['should'].append(
{'term': {
'time2': 'today'
}})
self.assertEqual(self.esbody.query_build.get_body(), self.testquery)
def test_update_aggs(self):
self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs)
self.esbody.groupby('ip')
self.testaggs['aggs'] = {
'ip-terms': {
'terms': {
'field': 'ip',
'size': 0
}
}
}
self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs)
self.esbody.groupby_date('date', '1h')\
.avg('count')
self.testaggs['aggs']['ip-terms']['aggs'] = {
'date-date_histogram': {
'date_histogram': {
'field': 'date',
'interval': '1h'
}
},
'count-avg': {
'avg': {
'field': 'count'
}
}
}
self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs)
def test_bodybuild(self):
body = {}
body.update(self.testquery)
body.update(self.testaggs)
self.assertEqual(self.esbody.get_body(), body)
self.esbody.range('time', 'yesterday', 'today')\
.should('time1', 'today')
self.testquery['query']['bool']['must'].append(
{'range': {
'time': {
'gte': 'yesterday',
'lte': 'today'
}
}})
self.testquery['query']['bool']['must'][0]['bool']['should'].append(
{'term': {
'time1': 'today'
}})
self.esbody.groupby_date('date', '1h')\
.avg('count')
self.testaggs['aggs'] = {
'date-date_histogram': {
'date_histogram': {
'field': 'date',
'interval': '1h'
}
},
'count-avg': {
'avg': {
'field': 'count'
}
}
}
body = {}
body.update(self.testquery)
body.update(self.testaggs)
self.assertEqual(self.esbody.get_body(), body)
class Test_SearchBodyBuild(unittest.TestCase):
def setUp(self):
self.esbody = SearchBodyBuild()
self.testquery = {
'query': {
'bool': {
'must': [
{
'bool': {
'should': []
}
},
]
}
}
}
self.testaggs = {}
def test_select_must(self):
self.esbody.must('time1', 'today')
self.testquery['query']['bool']['must'].append({'term': {'time1': 'today'}})
self.assertEqual(
self.esbody.query_build.get_body(), self.testquery
)
self.esbody.must('time2', 'today')
self.testquery['query']['bool']['must'].append({'term': {'time2': 'today'}})
self.assertEqual(
self.esbody.query_build.get_body(), self.testquery
)
def test_select_should(self):
self.esbody.should('time1', 'today')\
.should('time2', 'today')
self.testquery['query']['bool']['must'][0]['bool']['should'].append({'term': {'time1': 'today'}})
self.testquery['query']['bool']['must'][0]['bool']['should'].append({'term': {'time2': 'today'}})
self.assertEqual(
self.esbody.query_build.get_body(), self.testquery
)
def test_select_range(self):
self.esbody.range('time', 'yesterday', 'today')
self.testquery['query']['bool']['must'].append({'range': {'time': {'gte': 'yesterday', 'lte': 'today'}}})
self.assertEqual(
self.esbody.query_build.get_body(), self.testquery
)
def test_select_range_should(self):
self.esbody.range('time', 'yesterday', 'today')\
.should('time1', 'today')\
.should('time2', 'today')
self.testquery['query']['bool']['must'].append({'range': {'time': {'gte': 'yesterday', 'lte': 'today'}}})
self.testquery['query']['bool']['must'][0]['bool']['should'].append({'term': {'time1': 'today'}})
self.testquery['query']['bool']['must'][0]['bool']['should'].append({'term': {'time2': 'today'}})
self.assertEqual(
self.esbody.query_build.get_body(), self.testquery
)
def test_update_aggs(self):
self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs)
self.esbody.groupby('ip')
self.testaggs['aggs'] = {'ip-terms': {'terms': {'field': 'ip', 'size': 0}}}
self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs)
self.esbody.groupby_date('date', '1h')\
.avg('count')
self.testaggs['aggs']['ip-terms']['aggs'] = {
'date-date_histogram': {'date_histogram': {'field': 'date', 'interval': '1h'}},
'count-avg': {'avg': {'field': 'count'}}
}
self.assertEqual(self.esbody.aggs_build.get_body(), self.testaggs)
def test_bodybuild(self):
body = {}
body.update(self.testquery)
body.update(self.testaggs)
self.assertEqual(self.esbody.get_body(), body)
self.esbody.range('time', 'yesterday', 'today')\
.should('time1', 'today')
self.testquery['query']['bool']['must'].append({'range': {'time': {'gte': 'yesterday', 'lte': 'today'}}})
self.testquery['query']['bool']['must'][0]['bool']['should'].append({'term': {'time1': 'today'}})
self.esbody.groupby_date('date', '1h')\
.avg('count')
self.testaggs['aggs'] = {
'date-date_histogram': {'date_histogram': {'field': 'date', 'interval': '1h'}},
'count-avg': {'avg': {'field': 'count'}}
}
body = {}
body.update(self.testquery)
body.update(self.testaggs)
self.assertEqual(self.esbody.get_body(), body)
import datetime
from esearch.query import SearchBodyBuild
from esearch import parse_aggs
# Construct the search body
body_build = SearchBodyBuild()
end = datetime.datetime.now()
start = end - datetime.timedelta(days=10)
body = body_build.range('time', start, end)\
.must('ip', '10.10.0.0')\
.groupby_date('time', '1d')\
.groupby('domain')\
.sum('pv')\
.get_body()
print body
# Search
# import elasticsearch
# es_client = elasticsearch.Elasticsearch()
# res = es_client.search(index='test', body=body, doc_type='test')
res = {
'hits': {},
'_shards': {},
'took': 100,
'aggregations': {
'time-date_histogram': {
'buckets': [{
'domain-terms': {
'buckets': [{
class Test_ParseAggs(unittest.TestCase):
def setUp(self):
self.body_build = SearchBodyBuild()
self.body_build.range('time', 'start', 'end')\
.must('ip', '10.10.0.0')\
.groupby_date('time', '1d')\
.groupby('domain')\
.sum('pv')
self.res = {
'hits': {},
'_shards': {},
'took': 100,
'aggregations': {
'time-date_histogram': {
'buckets': [
{
'domain-terms': {
'buckets': [
{
'pv-sum': {
'value': 2000
},
'key': 'www.xxxx.com',
'doc_count': 1000
}
],
'sum_other_doc_count': 0,
'doc_count_error_upper_bound': 0
},
'key_as_string': '1111-11-11',
'key': 11111111,
'doc_count': 11111111
},
{
'domain-terms': {
'buckets': [
{
'pv-sum': {
'value': 2000
},
'key': 'www.xxxx.com',
'doc_count': 1000
}
],
'sum_other_doc_count': 0,
'doc_count_error_upper_bound': 0
},
'key_as_string': '2222-22-22',
'key': 22222222,
'doc_count': 22222222
}
],
}
}
}
self.pre_parser = parse_aggs.parse(self.body_build, self.res)
def get_date_and_value(self, idx):
idx = str(idx)
return '%s-%s-%s' % (idx*4, idx*2, idx*2), int(idx*8)
def test_get_buckets_metric(self):
for idx, time_bucket in enumerate(self.pre_parser.get_buckets('time')):
date, value = self.get_date_and_value(idx+1)
self.assertEqual(time_bucket.get_key_as_string(), date)
self.assertEqual(time_bucket.get_key(), value)
self.assertEqual(time_bucket.get_doc_count(), value)
for domain_bucket in time_bucket.get_buckets('domain'):
self.assertEqual(domain_bucket.get_key(), 'www.xxxx.com')
self.assertEqual(domain_bucket.get_doc_count(), 1000)
self.assertEqual(domain_bucket.get_sum('pv'), 2000)
with self.assertRaises(Exception):
domain_bucket.get_buckets('xxxxx')
import datetime
from esearch.query import SearchBodyBuild
from esearch import parse_aggs
# Construct the search body
body_build = SearchBodyBuild()
end = datetime.datetime.now()
start = end - datetime.timedelta(days=10)
body = body_build.range('time', start, end)\
.must('ip', '10.10.0.0')\
.groupby_date('time', '1d')\
.groupby('domain')\
.sum('pv')\
.get_body()
print body
# Search
# import elasticsearch
# es_client = elasticsearch.Elasticsearch()
# res = es_client.search(index='test', body=body, doc_type='test')
res = {
'hits': {},
'_shards': {},
'took': 100,
'aggregations': {
'time-date_histogram': {
'buckets': [
