def test_range_args(self):
filter_dsl = filters.filter_dsl_from_multidict(self.args)
assert ('range' in filter_dsl[1])
assert ('date' in filter_dsl[1]['range'])
assert ('comment_count' in filter_dsl[1]['range'])
assert ('2014-6-1' == filter_dsl[1]['range']['date']['lte'])
assert ('100' == filter_dsl[1]['range']['comment_count']['gt'])
def search_with_url_arguments(self, **kwargs):
# TODO: DRY with above
# TODO: Seriously.
query_dict = json.loads(file(self.filename).read())
query_dict.update(kwargs)
pagenum =1
if 'sort' not in query_dict:
query_dict['sort'] = "date:desc"
request = flask.request
filters = filter_dsl_from_multidict(request.args)
args_flat = request.args.to_dict(flat=True)
if 'page' in args_flat:
args_flat['from_'] = int(query_dict.get('size', '10')) * (int(args_flat['page'])-1)
pagenum = int(args_flat['page'])
query_dict.update(args_flat)
final_query_dict = dict((k, v) for (k, v) in query_dict.items() if k in ALLOWED_SEARCH_PARAMS)
final_query_dict['index'] = self.es_index
query_body = {"query": {'filtered':{}}}
query_body['query']['filtered']['filter'] = filters
final_query_dict['body'] = query_body
response = self.es.search(**final_query_dict)
response['query']= query_dict
return QueryResults(response,pagenum )
def test_range_args(self):
filter_dsl = filters.filter_dsl_from_multidict(self.args)
assert('range' in filter_dsl[1])
assert('date' in filter_dsl[1]['range'])
assert('comment_count' in filter_dsl[1]['range'])
assert('2014-6-1' == filter_dsl[1]['range']['date']['lte'])
assert('100' == filter_dsl[1]['range']['comment_count']['gt'])
def search_with_url_arguments(self, **kwargs):
# TODO: DRY with above
# TODO: Seriously.
query_dict = json.loads(file(self.filename).read())
query_dict.update(kwargs)
pagenum = 1
if 'sort' not in query_dict:
query_dict['sort'] = "date:desc"
request = flask.request
filters = filter_dsl_from_multidict(request.args)
args_flat = request.args.to_dict(flat=True)
if 'page' in args_flat:
args_flat['from_'] = int(query_dict.get(
'size', '10')) * (int(args_flat['page']) - 1)
pagenum = int(args_flat['page'])
query_dict.update(args_flat)
final_query_dict = {
k: v
for k, v in query_dict.items() if k in ALLOWED_SEARCH_PARAMS
}
final_query_dict['index'] = self.es_index
query_body = {"query": {'filtered': {}}}
query_body['query']['filtered']['filter'] = filters
final_query_dict['body'] = query_body
response = self.es.search(**final_query_dict)
response['query'] = query_dict
return QueryResults(response, pagenum)
def test_args_to_filter_dsl(self):
filter_dsl = filters.filter_dsl_from_multidict(self.args)
assert ('bool' in filter_dsl[0])
assert ('must' in filter_dsl[0]['bool'])
value1 = filter_dsl[0]['bool']['should'][0]['term']['category']
value2 = filter_dsl[0]['bool']['should'][1]['term']['category']
assert ('cats' in [value1, value2])
assert ('dogs' in [value1, value2])
def test_args_to_filter_dsl(self):
filter_dsl = filters.filter_dsl_from_multidict(self.args)
assert('bool' in filter_dsl['and'][0])
assert('must' in filter_dsl['and'][0]['bool'])
value1 = filter_dsl['and'][0]['bool']['should'][0]['term']['category']
value2 = filter_dsl['and'][0]['bool']['should'][1]['term']['category']
assert('cats' in [value1, value2])
assert('dogs' in [value1, value2])
def test_args_to_filter_dsl(self):
filter_dsl = filters.filter_dsl_from_multidict(self.args)
print filter_dsl
assert('and' in filter_dsl[0])
assert('or' in filter_dsl[0]['and'][0])
value1 = filter_dsl[0]['and'][0]['or'][0]['term']['category']
value2 = filter_dsl[0]['and'][0]['or'][1]['term']['category']
assert('cats' in [value1, value2])
assert('dogs' in [value1, value2])
assert('earth' in filter_dsl[0]['and'][1]['or'][0]['term']['planet'])
def test_args_to_filter_dsl(self):
filter_dsl = filters.filter_dsl_from_multidict(self.args)
print filter_dsl
assert ('and' in filter_dsl[0])
assert ('or' in filter_dsl[0]['and'][0])
value1 = filter_dsl[0]['and'][0]['or'][0]['term']['category']
value2 = filter_dsl[0]['and'][0]['or'][1]['term']['category']
assert ('cats' in [value1, value2])
assert ('dogs' in [value1, value2])
assert ('earth' in filter_dsl[0]['and'][1]['or'][0]['term']['planet'])
def search_with_url_arguments(self, term_facets=None, **kwargs):
query_file = json.loads(file(self.filename).read())
query_dict = query_file['query']
query_dict.update(kwargs)
pagenum = 1
request = flask.request
url_filters = filter_dsl_from_multidict(request.args)
args_flat = request.args.to_dict(flat=True)
query_body = {}
if term_facets:
facets_dsl = {}
if type(term_facets) is str:
term_facets = [term_facets] # so we can treat it as a list
for fieldname in term_facets:
facets_dsl[fieldname] = {
"terms": {
"field": fieldname,
"size": 10000
}
}
query_body["facets"] = facets_dsl
else:
if 'page' in args_flat:
args_flat['from_'] = int(query_dict.get(
'size', '10')) * (int(args_flat['page']) - 1)
pagenum = int(args_flat['page'])
args_flat_filtered = dict([(k, v) for k, v in args_flat.items()
if v])
query_dict.update(args_flat_filtered)
query_body['query'] = {'filtered': {'filter': {}}}
if url_filters:
query_body['query']['filtered']['filter']['and'] = [
f for f in url_filters
]
if 'filters' in query_file:
if 'and' not in query_body['query']['filtered']['filter']:
query_body['query']['filtered']['filter']['and'] = []
for json_filter in query_file['filters']:
query_body['query']['filtered']['filter']['and'].append(
json_filter)
final_query_dict = dict((k, v) for (k, v) in query_dict.items()
if k in ALLOWED_SEARCH_PARAMS)
final_query_dict['index'] = self.es_index
final_query_dict['body'] = query_body
response = self.es.search(**final_query_dict)
response['query'] = query_dict
return QueryResults(response, pagenum)
def search_with_url_arguments(self, term_facets=None, **kwargs):
query_file = json.loads(file(self.filename).read())
query_dict = query_file['query']
query_dict.update(kwargs)
pagenum = 1
request = flask.request
url_filters = filter_dsl_from_multidict(request.args)
args_flat = request.args.to_dict(flat=True)
query_body = {}
if term_facets:
facets_dsl = {}
if type(term_facets) is str:
term_facets = [term_facets] # so we can treat it as a list
for fieldname in term_facets:
facets_dsl[fieldname] = {
"terms": {"field": fieldname, "size": 10000}}
query_body["facets"] = facets_dsl
else:
if 'page' in args_flat:
args_flat['from_'] = int(
query_dict.get('size', '10')) * (int(args_flat['page']) - 1)
pagenum = int(args_flat['page'])
args_flat_filtered = dict(
[(k, v) for k, v in args_flat.items() if v])
query_dict.update(args_flat_filtered)
query_body['query'] = {'filtered': {'filter': {}}}
if url_filters:
query_body['query']['filtered']['filter'][
'and'] = [f for f in url_filters]
if 'filters' in query_file:
if 'and' not in query_body['query']['filtered']['filter']:
query_body['query']['filtered']['filter']['and'] = []
for json_filter in query_file['filters']:
query_body['query']['filtered'][
'filter']['and'].append(json_filter)
final_query_dict = dict((k, v)
for (k, v) in query_dict.items() if k in ALLOWED_SEARCH_PARAMS)
final_query_dict['index'] = self.es_index
final_query_dict['body'] = query_body
response = self.es.search(**final_query_dict)
response['query'] = query_dict
return QueryResults(response, pagenum)
def test_default_days_incorrect_range(self):
args = MultiDict([('filter_range_date_gte', '2014-6'),
('filter_range_date_lte', '2014-1')])
filter_dsl = filters.filter_dsl_from_multidict(args)
assert (filter_dsl[0]['range']['date']['gte'] == '2014-1-1')
assert (filter_dsl[0]['range']['date']['lte'] == '2014-6-30')
def test_date_validation_with_days_correct_range(self):
args = MultiDict([('filter_range_date_gte', '2014-1-23'),
('filter_range_date_lte', '2014-6-23')])
filter_dsl = filters.filter_dsl_from_multidict(args)
assert (filter_dsl[0]['range']['date']['gte'] == '2014-1-23')
assert (filter_dsl[0]['range']['date']['lte'] == '2014-6-23')
def search_with_url_arguments(self, aggregations=None, **kwargs):
query_file = json.loads(file(self.filename).read())
query_dict = query_file['query']
'''
These dict constructors split the kwargs from the template into filter
arguments and arguments that can be placed directly into the query body.
The dict constructor syntax supports python 2.6, 2.7, and 3.x
If python 2.7, use dict comprehension and iteritems()
With python 3, use dict comprehension and items() (items() replaces
iteritems and is just as fast)
'''
filter_args = dict((key, value) for (key, value) in kwargs.items()
if key.startswith('filter_'))
non_filter_args = dict((key, value) for (key, value) in kwargs.items()
if not key.startswith('filter_'))
query_dict.update(non_filter_args)
pagenum = 1
request = flask.request
# Add in filters from the template.
new_multidict = request.args.copy()
for key, value in filter_args.items():
new_multidict.add(key, value)
url_filters = filter_dsl_from_multidict(new_multidict)
args_flat = request.args.to_dict(flat=True)
query_body = {}
if aggregations:
aggs_dsl = {}
if type(aggregations) is str:
aggregations = [aggregations] # so we can treat it as a list
for fieldname in aggregations:
aggs_dsl[fieldname] = {'terms':
{'field': fieldname, 'size': 10000}}
query_body['aggs'] = aggs_dsl
else:
if 'page' in args_flat:
args_flat['from_'] = int(
query_dict.get('size', '10')) * (int(args_flat['page']) - 1)
pagenum = int(args_flat['page'])
args_flat_filtered = dict(
[(k, v) for k, v in args_flat.items() if v])
query_dict.update(args_flat_filtered)
query_body['query'] = {'filtered': {'filter': {}}}
if url_filters:
query_body['query']['filtered']['filter'][
'and'] = [f for f in url_filters]
if 'filters' in query_file:
if 'and' not in query_body['query']['filtered']['filter']:
query_body['query']['filtered']['filter']['and'] = []
for json_filter in query_file['filters']:
query_body['query']['filtered'][
'filter']['and'].append(json_filter)
final_query_dict = dict((k, v)
for (k, v) in query_dict.items() if k in ALLOWED_SEARCH_PARAMS)
final_query_dict['index'] = self.es_index
final_query_dict['body'] = query_body
response = self.es.search(**final_query_dict)
response['query'] = query_dict
return QueryResults(response, pagenum)
def test_default_days_incorrect_range(self):
args = MultiDict([('filter_range_date_gte', '2014-6'),
('filter_range_date_lte', '2014-1')])
filter_dsl = filters.filter_dsl_from_multidict(args)
assert(filter_dsl[0]['range']['date']['gte'] == '2014-1-1')
assert(filter_dsl[0]['range']['date']['lte'] == '2014-6-30')
def test_date_validation_with_days_correct_range(self):
args = MultiDict([('filter_range_date_gte', '2014-1-23'),
('filter_range_date_lte', '2014-6-23')])
filter_dsl = filters.filter_dsl_from_multidict(args)
assert(filter_dsl[0]['range']['date']['gte'] == '2014-1-23')
assert(filter_dsl[0]['range']['date']['lte'] == '2014-6-23')
def search(self, aggregations=None, use_url_arguments=True, **kwargs):
query_file = json.loads(file(self.filename).read())
query_dict = query_file['query']
'''
These dict constructors split the kwargs from the template into filter
arguments and arguments that can be placed directly into the query body.
The dict constructor syntax supports python 2.6, 2.7, and 3.x
If python 2.7, use dict comprehension and iteritems()
With python 3, use dict comprehension and items() (items() replaces
iteritems and is just as fast)
'''
filter_args = dict((key, value) for (key, value) in kwargs.items()
if key.startswith('filter_'))
non_filter_args = dict((key, value) for (key, value) in kwargs.items()
if not key.startswith('filter_'))
query_dict.update(non_filter_args)
pagenum = 1
request = flask.request
# Add in filters from the template.
new_multidict = MultiDict()
# First add the url arguments if requested
if use_url_arguments:
new_multidict = request.args.copy()
# Next add the arguments from the search() function used in the
# template
for key, value in filter_args.items():
new_multidict.add(key, value)
filters = filter_dsl_from_multidict(new_multidict)
args_flat = request.args.to_dict(flat=True)
query_body = {}
if aggregations:
aggs_dsl = {}
if type(aggregations) is str:
aggregations = [aggregations] # so we can treat it as a list
for fieldname in aggregations:
aggs_dsl[fieldname] = {
'terms': {
'field': fieldname,
'size': 10000
}
}
query_body['aggs'] = aggs_dsl
else:
if 'page' in args_flat:
args_flat['from_'] = int(query_dict.get(
'size', '10')) * (int(args_flat['page']) - 1)
pagenum = int(args_flat['page'])
args_flat_filtered = dict([(k, v) for k, v in args_flat.items()
if v])
query_dict.update(args_flat_filtered)
query_body['query'] = {'filtered': {'filter': {}}}
if filters:
query_body['query']['filtered']['filter']['and'] = [
f for f in filters
]
if 'filters' in query_file:
if 'and' not in query_body['query']['filtered']['filter']:
query_body['query']['filtered']['filter']['and'] = []
for json_filter in query_file['filters']:
query_body['query']['filtered']['filter']['and'].append(
json_filter)
final_query_dict = dict((k, v) for (k, v) in query_dict.items()
if k in ALLOWED_SEARCH_PARAMS)
final_query_dict['index'] = self.es_index
final_query_dict['body'] = query_body
response = self.es.search(**final_query_dict)
response['query'] = query_dict
return QueryResults(response, pagenum)
