def test_apply_insane_params(self):
# this test case shouldn't raise exception
qf = QueryFilter()
qf.add_filter(FacetFilter('test'))
qf.add_filter(
PivotFilter(
'pivot_test',
FacetPivotFilter('a'),
FacetPivotFilter('b'),
)
)
qf.add_ordering(
OrderingFilter(
'sort',
OrderingValue('-score', '-score'),
OrderingValue('price', 'price'),
default='-score',
)
)
q = self.searcher.search()
params = {
111: 222,
'\ufffd': '',
'\uffff'.encode('utf-8'): '',
'test': ['\ufffd', '\uffff'.encode('utf-8')],
'pivot_test': ['\ufffd', '\uffff'.encode('utf-8')],
'sort': ['\ufffd', '\uffff'.encode('utf-8')],
}
q = qf.apply(q, params)
def test_apply_insane_params(self):
# this test case shouldn't raise exception
qf = QueryFilter()
qf.add_filter(FacetFilter("test"))
qf.add_filter(PivotFilter("pivot_test", FacetPivotFilter("a"), FacetPivotFilter("b")))
qf.add_ordering(
OrderingFilter("sort", OrderingValue("-score", "-score"), OrderingValue("price", "price"), default="-score")
)
q = self.searcher.search()
params = {
111: 222,
"\ufffd": "",
"\uffff".encode("utf-8"): "",
"test": ["\ufffd", "\uffff".encode("utf-8")],
"pivot_test": ["\ufffd", "\uffff".encode("utf-8")],
"sort": ["\ufffd", "\uffff".encode("utf-8")],
}
q = qf.apply(q, params)
def test_ordering_filter(self):
with self.patch_send_request() as send_request:
send_request.return_value = '''
{
"facet_counts": {
"facet_queries": {
"date_created__today": 28,
"date_created__week_ago": 105,
"dist__d5": 0,
"dist__d10": 12,
"dist__d20": 40
},
"facet_fields": {},
"facet_dates":{},
"facet_ranges":{}
}
}'''
q = self.searcher.search()
qf = QueryFilter()
qf.add_ordering(
OrderingFilter(
'sort',
OrderingValue('-score', '-score'),
OrderingValue('price', 'price'),
OrderingValue('-price', '-price')))
params = {
'sort': '-price',
}
q = qf.apply(q, params)
raw_query = force_unicode(q)
self.assertIn('sort=price desc', raw_query)
ordering_filter = qf.ordering_filter
self.assertEqual(ordering_filter.get_value('-price').selected, True)
self.assertEqual(ordering_filter.get_value('-price').direction, OrderingValue.DESC)
def test_ordering_filter(self):
with self.patch_send_request() as send_request:
send_request.return_value = """
{
"facet_counts": {
"facet_queries": {
"date_created__today": 28,
"date_created__week_ago": 105,
"dist__d5": 0,
"dist__d10": 12,
"dist__d20": 40
},
"facet_fields": {},
"facet_dates":{},
"facet_ranges":{}
}
}"""
q = self.searcher.search()
qf = QueryFilter()
qf.add_ordering(
OrderingFilter(
"sort",
OrderingValue("-score", "-score"),
OrderingValue("price", "price"),
OrderingValue("-price", "-price"),
)
)
params = {"sort": "-price"}
q = qf.apply(q, params)
raw_query = force_unicode(q)
self.assertIn("sort=price desc", raw_query)
ordering_filter = qf.ordering_filter
self.assertEqual(ordering_filter.get_value("-price").selected, True)
self.assertEqual(ordering_filter.get_value("-price").direction, OrderingValue.DESC)
def test_queryfilter(self):
with self.patch_send_request() as send_request:
send_request.return_value = '''{
"grouped":{
"company":{
"matches":0,
"ngroups":0,
"groups":[]}},
"facet_counts":{
"facet_queries":{
"date_created__today":28,
"date_created__week_ago":105,
"dist__d5":0,
"dist__d10":12,
"dist__d20":40},
"facet_fields":{
"cat":[
"100",500,
"5",10,
"2",5,
"1",2,
"13",1]},
"facet_dates":{},
"facet_ranges":{}}}'''
q = self.searcher.search()
qf = QueryFilter()
qf.add_filter(
CategoryFilter(
'cat', 'category', mincount=1,
_local_params={'cache': False}))
qf.add_filter(Filter('country', select_multiple=False))
qf.add_filter(
FacetQueryFilter(
'date_created',
FacetQueryFilterValue(
'today',
X(date_created__gte='NOW/DAY-1DAY'),
title='Only new',
help_text='Documents one day later'),
FacetQueryFilterValue(
'week_ago',
X(date_created__gte='NOW/DAY-7DAY'),
title='Week ago')))
qf.add_filter(RangeFilter('price', 'price_unit', gather_stats=True,
_local_params=LocalParams(cache=False),
type=Integer))
qf.add_filter(
FacetQueryFilter(
'dist',
FacetQueryFilterValue(
'd5',
None, _local_params=LocalParams('geofilt', d=5)),
FacetQueryFilterValue(
'd10',
None, _local_params=LocalParams('geofilt', d=10)),
FacetQueryFilterValue(
'd20',
None, _local_params=LocalParams('geofilt', d=20)),
select_multiple=False))
qf.add_ordering(
OrderingFilter(
'sort',
OrderingValue('-score', '-score'),
OrderingValue('price', 'price'),
OrderingValue('-price', '-price')))
params = {
'cat': ['5', '13'],
'country': ['us', 'ru'],
'date_created': 'today',
'price__gte': '100',
'price__lte': ['200', 'nan'],
'dist': 'd10',
'sort': '-price',
}
q = qf.apply(q, params)
raw_query = str(q)
self.assertIn('facet=true', raw_query)
self.assertIn('facet.field={!cache=false key=cat ex=cat}category', raw_query)
self.assertIn('facet.query={!key=date_created__today ex=date_created}date_created:[NOW/DAY-1DAY TO *]', raw_query)
self.assertIn('facet.query={!key=date_created__week_ago ex=date_created}date_created:[NOW/DAY-7DAY TO *]', raw_query)
self.assertIn('facet.query={!geofilt d=5 key=dist__d5 ex=dist}', raw_query)
self.assertIn('facet.query={!geofilt d=10 key=dist__d10 ex=dist}', raw_query)
self.assertIn('facet.query={!geofilt d=20 key=dist__d20 ex=dist}', raw_query)
# self.assertIn('stats=true', raw_query)
# self.assertIn('stats.field=price_unit', raw_query)
self.assertIn('fq={!cache=false tag=cat}(category:"5" OR category:"13")', raw_query)
self.assertIn('fq={!tag=country}country:"ru"', raw_query)
self.assertIn('fq={!tag=date_created}date_created:[NOW/DAY-1DAY TO *]', raw_query)
self.assertIn('fq={!cache=false tag=price}price_unit:[100 TO *] AND price_unit:[* TO 200]', raw_query)
self.assertIn('fq={!geofilt d=10 tag=dist}', raw_query)
self.assertIn('sort=price desc', raw_query)
results = q.results
with self.patch_send_request() as send_request:
send_request.return_value = '''{
"response":{"numFound":800,"start":0,"docs":[]
},
"stats":{
"stats_fields":{
"price_unit":{
"min":3.5,
"max":892.0,
"count":1882931,
"missing":556686,
"sum":5.677964302447648E13,
"sumOfSquares":2.452218850256837E26,
"mean":3.0154924967763808E7,
"stddev":1.1411980204045008E10}}}}'''
qf.process_results(results)
category_filter = qf.get_filter('cat')
self.assertIsInstance(category_filter, CategoryFilter)
self.assertEqual(category_filter.name, 'cat')
self.assertEqual(category_filter.field, 'category')
self.assertEqual(category_filter.all_values[0].value, '100')
self.assertEqual(category_filter.all_values[0].count, 500)
self.assertEqual(category_filter.all_values[0].selected, False)
self.assertEqual(category_filter.all_values[0].title, '100')
self.assertEqual(category_filter.all_values[1].value, '5')
self.assertEqual(category_filter.all_values[1].count, 10)
self.assertEqual(category_filter.all_values[1].selected, True)
self.assertEqual(category_filter.all_values[2].value, '2')
self.assertEqual(category_filter.all_values[2].count, 5)
self.assertEqual(category_filter.all_values[2].selected, False)
self.assertEqual(category_filter.all_values[3].value, '1')
self.assertEqual(category_filter.all_values[3].count, 2)
self.assertEqual(category_filter.all_values[3].selected, False)
self.assertEqual(category_filter.all_values[4].value, '13')
self.assertEqual(category_filter.all_values[4].count, 1)
self.assertEqual(category_filter.all_values[4].selected, True)
price_filter = qf.get_filter('price')
self.assertEqual(price_filter.from_value, 100)
self.assertEqual(price_filter.to_value, 200)
self.assertEqual(price_filter.min, 3.5)
self.assertEqual(price_filter.max, 892.0)
date_created_filter = qf.get_filter('date_created')
self.assertEqual(date_created_filter.get_value('today').count, 28)
self.assertEqual(date_created_filter.get_value('today').selected, True)
self.assertEqual(date_created_filter.get_value('today').title, 'Only new')
self.assertEqual(date_created_filter.get_value('today').opts['help_text'], 'Documents one day later')
self.assertEqual(date_created_filter.get_value('week_ago').count, 105)
dist_filter = qf.get_filter('dist')
self.assertEqual(dist_filter.get_value('d5').count, 0)
self.assertEqual(dist_filter.get_value('d5').selected, False)
self.assertEqual(dist_filter.get_value('d10').count, 12)
self.assertEqual(dist_filter.get_value('d10').selected, True)
self.assertEqual(dist_filter.get_value('d20').count, 40)
self.assertEqual(dist_filter.get_value('d20').selected, False)
ordering_filter = qf.ordering_filter
self.assertEqual(ordering_filter.get_value('-price').selected, True)
self.assertEqual(ordering_filter.get_value('-price').direction, OrderingValue.DESC)
def test_facet_query_filter(self):
with self.patch_send_request() as send_request:
send_request.return_value = '''
{
"facet_counts": {
"facet_queries": {
"date_created__today": 28,
"date_created__week_ago": 105,
"dist__d5": 0,
"dist__d10": 12,
"dist__d20": 40
},
"facet_fields": {},
"facet_dates":{},
"facet_ranges":{}
}
}'''
q = self.searcher.search()
qf = QueryFilter()
qf.add_filter(
FacetQueryFilter(
'date_created',
FacetQueryFilterValue(
'today',
X(date_created__gte='NOW/DAY-1DAY'),
_local_params=LocalParams(ex='test'),
title='Only new',
help_text='Documents one day later'),
FacetQueryFilterValue(
'week_ago',
X(date_created__gte='NOW/DAY-7DAY'),
title='Week ago')))
qf.add_filter(
FacetQueryFilter(
'dist',
FacetQueryFilterValue(
'd5',
None, _local_params=LocalParams('geofilt', d=5, tag='d5')),
FacetQueryFilterValue(
'd10',
None, _local_params=LocalParams('geofilt', d=10, tag='d10')),
FacetQueryFilterValue(
'd20',
None, _local_params=LocalParams('geofilt', d=20, tag='d20')),
select_multiple=False))
qf.add_ordering(
OrderingFilter(
'sort',
OrderingValue('-score', '-score'),
OrderingValue('price', 'price'),
OrderingValue('-price', '-price')))
params = {
'cat': ['5', '13'],
'country': ['us', 'ru'],
'date_created': 'today',
'dist': 'd10',
}
q = qf.apply(q, params)
raw_query = force_unicode(q)
self.assertIn('facet=true', raw_query)
self.assertIn('facet.query={!ex=test,date_created key=date_created__today}date_created:[NOW/DAY-1DAY TO *]', raw_query)
self.assertIn('facet.query={!key=date_created__week_ago ex=date_created}date_created:[NOW/DAY-7DAY TO *]', raw_query)
self.assertIn('facet.query={!geofilt d=5 tag=d5 key=dist__d5 ex=dist}', raw_query)
self.assertIn('facet.query={!geofilt d=10 tag=d10 key=dist__d10 ex=dist}', raw_query)
self.assertIn('facet.query={!geofilt d=20 tag=d20 key=dist__d20 ex=dist}', raw_query)
self.assertIn('fq={!tag=date_created}date_created:[NOW/DAY-1DAY TO *]', raw_query)
self.assertIn('fq={!geofilt d=10 tag=d10,dist}', raw_query)
qf.process_results(q.results)
date_created_filter = qf.get_filter('date_created')
self.assertEqual(date_created_filter.get_value('today').count, 28)
self.assertEqual(date_created_filter.get_value('today').count_plus, '28')
self.assertEqual(date_created_filter.get_value('today').selected, True)
self.assertEqual(date_created_filter.get_value('today').title, 'Only new')
self.assertEqual(date_created_filter.get_value('today').opts['help_text'], 'Documents one day later')
self.assertEqual(date_created_filter.get_value('week_ago').count, 105)
self.assertEqual(date_created_filter.get_value('week_ago').count_plus, '105')
dist_filter = qf.get_filter('dist')
self.assertEqual(dist_filter.get_value('d5').count, 0)
self.assertEqual(dist_filter.get_value('d5').selected, False)
self.assertEqual(dist_filter.get_value('d10').count, 12)
self.assertEqual(dist_filter.get_value('d10').selected, True)
self.assertEqual(dist_filter.get_value('d20').count, 40)
self.assertEqual(dist_filter.get_value('d20').selected, False)
def tasks(project):
if project not in app.config['CELERIUM_PROJECTS']:
abort(404)
try:
page = int(request.args.get('page', '1'))
except ValueError:
page = 1
search_query = (
task_searcher.search()
.filter(project=project))
fixed_dt = request.args.get('fixed_dt')
if fixed_dt:
try:
fixed_dt = datetime.strptime(fixed_dt, "%Y-%m-%d %H:%M:%S.%f")
search_query = search_query.filter(timestamp__lte=fixed_dt)
except ValueError:
fixed_dt = None
qf = QueryFilter()
qf.add_filter(FacetFilter('worker', 'worker_name', mincount=0))
qf.add_filter(FacetFilter('state', mincount=0))
qf.add_filter(FacetFilter('module', mincount=1))
if 'module' in request.args:
qf.add_filter(FacetFilter('name', mincount=1))
qf.add_filter(
FacetQueryFilter(
'timestamp',
FacetQueryFilterValue('last_hour', X(timestamp__gte='NOW-1HOUR'), title='Last Hour'),
FacetQueryFilterValue('today', X(timestamp__gte='NOW/DAY'), title='Today'),
FacetQueryFilterValue('last_day', X(timestamp__gte='NOW-1DAY'), title='Last Day')))
qf.add_filter(
FacetQueryFilter(
'runtime',
FacetQueryFilterValue('1m', X(runtime__gte=60), title='Minute'),
FacetQueryFilterValue('10m', X(runtime__gte=60 * 10), title='10 Minutes'),
FacetQueryFilterValue('1h', X(runtime__gte=60 * 60), title='Hour')))
qf.add_ordering(
OrderingFilter(
'sort',
OrderingValue(
'timestamp', 'timestamp',
),
OrderingValue(
'-timestamp', '-timestamp',
),
OrderingValue(
'runtime', 'runtime',
),
OrderingValue(
'-runtime', '-runtime',
),
default='-timestamp'))
search_query = qf.apply(search_query, request.args)
pagination = Pagination(search_query, page=page, per_page=20)
qf.process_results(pagination.query.results)
return render_template('tasks.html',
project=project,
pagination=pagination,
query_filter=qf,
fixed_dt=fixed_dt,
current_dt=datetime.now())
def test_facet_query_filter(self):
with self.patch_send_request() as send_request:
send_request.return_value = """
{
"facet_counts": {
"facet_queries": {
"date_created__today": 28,
"date_created__week_ago": 105,
"dist__d5": 0,
"dist__d10": 12,
"dist__d20": 40
},
"facet_fields": {},
"facet_dates":{},
"facet_ranges":{}
}
}"""
q = self.searcher.search()
qf = QueryFilter()
qf.add_filter(
FacetQueryFilter(
"date_created",
FacetQueryFilterValue(
"today",
X(date_created__gte="NOW/DAY-1DAY"),
_local_params=LocalParams(ex="test"),
title="Only new",
help_text="Documents one day later",
),
FacetQueryFilterValue("week_ago", X(date_created__gte="NOW/DAY-7DAY"), title="Week ago"),
)
)
qf.add_filter(
FacetQueryFilter(
"dist",
FacetQueryFilterValue("d5", None, _local_params=LocalParams("geofilt", d=5, tag="d5")),
FacetQueryFilterValue("d10", None, _local_params=LocalParams("geofilt", d=10, tag="d10")),
FacetQueryFilterValue("d20", None, _local_params=LocalParams("geofilt", d=20, tag="d20")),
select_multiple=False,
)
)
qf.add_ordering(
OrderingFilter(
"sort",
OrderingValue("-score", "-score"),
OrderingValue("price", "price"),
OrderingValue("-price", "-price"),
)
)
params = {"cat": ["5", "13"], "country": ["us", "ru"], "date_created": "today", "dist": "d10"}
q = qf.apply(q, params)
raw_query = force_unicode(q)
self.assertIn("facet=true", raw_query)
self.assertIn(
"facet.query={!ex=test,date_created key=date_created__today}date_created:[NOW/DAY-1DAY TO *]", raw_query
)
self.assertIn(
"facet.query={!key=date_created__week_ago ex=date_created}date_created:[NOW/DAY-7DAY TO *]", raw_query
)
self.assertIn("facet.query={!geofilt d=5 tag=d5 key=dist__d5 ex=dist}", raw_query)
self.assertIn("facet.query={!geofilt d=10 tag=d10 key=dist__d10 ex=dist}", raw_query)
self.assertIn("facet.query={!geofilt d=20 tag=d20 key=dist__d20 ex=dist}", raw_query)
self.assertIn("fq={!tag=date_created}date_created:[NOW/DAY-1DAY TO *]", raw_query)
self.assertIn("fq={!geofilt d=10 tag=d10,dist}", raw_query)
qf.process_results(q.results)
date_created_filter = qf.get_filter("date_created")
self.assertEqual(date_created_filter.get_value("today").count, 28)
self.assertEqual(date_created_filter.get_value("today").count_plus, "28")
self.assertEqual(date_created_filter.get_value("today").selected, True)
self.assertEqual(date_created_filter.get_value("today").title, "Only new")
self.assertEqual(date_created_filter.get_value("today").opts["help_text"], "Documents one day later")
self.assertEqual(date_created_filter.get_value("week_ago").count, 105)
self.assertEqual(date_created_filter.get_value("week_ago").count_plus, "105")
dist_filter = qf.get_filter("dist")
self.assertEqual(dist_filter.get_value("d5").count, 0)
self.assertEqual(dist_filter.get_value("d5").selected, False)
self.assertEqual(dist_filter.get_value("d10").count, 12)
self.assertEqual(dist_filter.get_value("d10").selected, True)
self.assertEqual(dist_filter.get_value("d20").count, 40)
self.assertEqual(dist_filter.get_value("d20").selected, False)
