def test_range_args(self): filter_dsl = filters.filter_dsl_from_multidict(self.args) assert ('range' in filter_dsl[1]) assert ('date' in filter_dsl[1]['range']) assert ('comment_count' in filter_dsl[1]['range']) assert ('2014-6-1' == filter_dsl[1]['range']['date']['lte']) assert ('100' == filter_dsl[1]['range']['comment_count']['gt'])
def search_with_url_arguments(self, **kwargs): # TODO: DRY with above # TODO: Seriously. query_dict = json.loads(file(self.filename).read()) query_dict.update(kwargs) pagenum =1 if 'sort' not in query_dict: query_dict['sort'] = "date:desc" request = flask.request filters = filter_dsl_from_multidict(request.args) args_flat = request.args.to_dict(flat=True) if 'page' in args_flat: args_flat['from_'] = int(query_dict.get('size', '10')) * (int(args_flat['page'])-1) pagenum = int(args_flat['page']) query_dict.update(args_flat) final_query_dict = dict((k, v) for (k, v) in query_dict.items() if k in ALLOWED_SEARCH_PARAMS) final_query_dict['index'] = self.es_index query_body = {"query": {'filtered':{}}} query_body['query']['filtered']['filter'] = filters final_query_dict['body'] = query_body response = self.es.search(**final_query_dict) response['query']= query_dict return QueryResults(response,pagenum )
def test_range_args(self): filter_dsl = filters.filter_dsl_from_multidict(self.args) assert('range' in filter_dsl[1]) assert('date' in filter_dsl[1]['range']) assert('comment_count' in filter_dsl[1]['range']) assert('2014-6-1' == filter_dsl[1]['range']['date']['lte']) assert('100' == filter_dsl[1]['range']['comment_count']['gt'])
def search_with_url_arguments(self, **kwargs): # TODO: DRY with above # TODO: Seriously. query_dict = json.loads(file(self.filename).read()) query_dict.update(kwargs) pagenum = 1 if 'sort' not in query_dict: query_dict['sort'] = "date:desc" request = flask.request filters = filter_dsl_from_multidict(request.args) args_flat = request.args.to_dict(flat=True) if 'page' in args_flat: args_flat['from_'] = int(query_dict.get( 'size', '10')) * (int(args_flat['page']) - 1) pagenum = int(args_flat['page']) query_dict.update(args_flat) final_query_dict = { k: v for k, v in query_dict.items() if k in ALLOWED_SEARCH_PARAMS } final_query_dict['index'] = self.es_index query_body = {"query": {'filtered': {}}} query_body['query']['filtered']['filter'] = filters final_query_dict['body'] = query_body response = self.es.search(**final_query_dict) response['query'] = query_dict return QueryResults(response, pagenum)
def test_args_to_filter_dsl(self): filter_dsl = filters.filter_dsl_from_multidict(self.args) assert ('bool' in filter_dsl[0]) assert ('must' in filter_dsl[0]['bool']) value1 = filter_dsl[0]['bool']['should'][0]['term']['category'] value2 = filter_dsl[0]['bool']['should'][1]['term']['category'] assert ('cats' in [value1, value2]) assert ('dogs' in [value1, value2])
def test_args_to_filter_dsl(self): filter_dsl = filters.filter_dsl_from_multidict(self.args) assert('bool' in filter_dsl['and'][0]) assert('must' in filter_dsl['and'][0]['bool']) value1 = filter_dsl['and'][0]['bool']['should'][0]['term']['category'] value2 = filter_dsl['and'][0]['bool']['should'][1]['term']['category'] assert('cats' in [value1, value2]) assert('dogs' in [value1, value2])
def test_args_to_filter_dsl(self): filter_dsl = filters.filter_dsl_from_multidict(self.args) print filter_dsl assert('and' in filter_dsl[0]) assert('or' in filter_dsl[0]['and'][0]) value1 = filter_dsl[0]['and'][0]['or'][0]['term']['category'] value2 = filter_dsl[0]['and'][0]['or'][1]['term']['category'] assert('cats' in [value1, value2]) assert('dogs' in [value1, value2]) assert('earth' in filter_dsl[0]['and'][1]['or'][0]['term']['planet'])
def test_args_to_filter_dsl(self): filter_dsl = filters.filter_dsl_from_multidict(self.args) print filter_dsl assert ('and' in filter_dsl[0]) assert ('or' in filter_dsl[0]['and'][0]) value1 = filter_dsl[0]['and'][0]['or'][0]['term']['category'] value2 = filter_dsl[0]['and'][0]['or'][1]['term']['category'] assert ('cats' in [value1, value2]) assert ('dogs' in [value1, value2]) assert ('earth' in filter_dsl[0]['and'][1]['or'][0]['term']['planet'])
def search_with_url_arguments(self, term_facets=None, **kwargs): query_file = json.loads(file(self.filename).read()) query_dict = query_file['query'] query_dict.update(kwargs) pagenum = 1 request = flask.request url_filters = filter_dsl_from_multidict(request.args) args_flat = request.args.to_dict(flat=True) query_body = {} if term_facets: facets_dsl = {} if type(term_facets) is str: term_facets = [term_facets] # so we can treat it as a list for fieldname in term_facets: facets_dsl[fieldname] = { "terms": { "field": fieldname, "size": 10000 } } query_body["facets"] = facets_dsl else: if 'page' in args_flat: args_flat['from_'] = int(query_dict.get( 'size', '10')) * (int(args_flat['page']) - 1) pagenum = int(args_flat['page']) args_flat_filtered = dict([(k, v) for k, v in args_flat.items() if v]) query_dict.update(args_flat_filtered) query_body['query'] = {'filtered': {'filter': {}}} if url_filters: query_body['query']['filtered']['filter']['and'] = [ f for f in url_filters ] if 'filters' in query_file: if 'and' not in query_body['query']['filtered']['filter']: query_body['query']['filtered']['filter']['and'] = [] for json_filter in query_file['filters']: query_body['query']['filtered']['filter']['and'].append( json_filter) final_query_dict = dict((k, v) for (k, v) in query_dict.items() if k in ALLOWED_SEARCH_PARAMS) final_query_dict['index'] = self.es_index final_query_dict['body'] = query_body response = self.es.search(**final_query_dict) response['query'] = query_dict return QueryResults(response, pagenum)
def search_with_url_arguments(self, term_facets=None, **kwargs): query_file = json.loads(file(self.filename).read()) query_dict = query_file['query'] query_dict.update(kwargs) pagenum = 1 request = flask.request url_filters = filter_dsl_from_multidict(request.args) args_flat = request.args.to_dict(flat=True) query_body = {} if term_facets: facets_dsl = {} if type(term_facets) is str: term_facets = [term_facets] # so we can treat it as a list for fieldname in term_facets: facets_dsl[fieldname] = { "terms": {"field": fieldname, "size": 10000}} query_body["facets"] = facets_dsl else: if 'page' in args_flat: args_flat['from_'] = int( query_dict.get('size', '10')) * (int(args_flat['page']) - 1) pagenum = int(args_flat['page']) args_flat_filtered = dict( [(k, v) for k, v in args_flat.items() if v]) query_dict.update(args_flat_filtered) query_body['query'] = {'filtered': {'filter': {}}} if url_filters: query_body['query']['filtered']['filter'][ 'and'] = [f for f in url_filters] if 'filters' in query_file: if 'and' not in query_body['query']['filtered']['filter']: query_body['query']['filtered']['filter']['and'] = [] for json_filter in query_file['filters']: query_body['query']['filtered'][ 'filter']['and'].append(json_filter) final_query_dict = dict((k, v) for (k, v) in query_dict.items() if k in ALLOWED_SEARCH_PARAMS) final_query_dict['index'] = self.es_index final_query_dict['body'] = query_body response = self.es.search(**final_query_dict) response['query'] = query_dict return QueryResults(response, pagenum)
def test_default_days_incorrect_range(self): args = MultiDict([('filter_range_date_gte', '2014-6'), ('filter_range_date_lte', '2014-1')]) filter_dsl = filters.filter_dsl_from_multidict(args) assert (filter_dsl[0]['range']['date']['gte'] == '2014-1-1') assert (filter_dsl[0]['range']['date']['lte'] == '2014-6-30')
def test_date_validation_with_days_correct_range(self): args = MultiDict([('filter_range_date_gte', '2014-1-23'), ('filter_range_date_lte', '2014-6-23')]) filter_dsl = filters.filter_dsl_from_multidict(args) assert (filter_dsl[0]['range']['date']['gte'] == '2014-1-23') assert (filter_dsl[0]['range']['date']['lte'] == '2014-6-23')
def search_with_url_arguments(self, aggregations=None, **kwargs): query_file = json.loads(file(self.filename).read()) query_dict = query_file['query'] ''' These dict constructors split the kwargs from the template into filter arguments and arguments that can be placed directly into the query body. The dict constructor syntax supports python 2.6, 2.7, and 3.x If python 2.7, use dict comprehension and iteritems() With python 3, use dict comprehension and items() (items() replaces iteritems and is just as fast) ''' filter_args = dict((key, value) for (key, value) in kwargs.items() if key.startswith('filter_')) non_filter_args = dict((key, value) for (key, value) in kwargs.items() if not key.startswith('filter_')) query_dict.update(non_filter_args) pagenum = 1 request = flask.request # Add in filters from the template. new_multidict = request.args.copy() for key, value in filter_args.items(): new_multidict.add(key, value) url_filters = filter_dsl_from_multidict(new_multidict) args_flat = request.args.to_dict(flat=True) query_body = {} if aggregations: aggs_dsl = {} if type(aggregations) is str: aggregations = [aggregations] # so we can treat it as a list for fieldname in aggregations: aggs_dsl[fieldname] = {'terms': {'field': fieldname, 'size': 10000}} query_body['aggs'] = aggs_dsl else: if 'page' in args_flat: args_flat['from_'] = int( query_dict.get('size', '10')) * (int(args_flat['page']) - 1) pagenum = int(args_flat['page']) args_flat_filtered = dict( [(k, v) for k, v in args_flat.items() if v]) query_dict.update(args_flat_filtered) query_body['query'] = {'filtered': {'filter': {}}} if url_filters: query_body['query']['filtered']['filter'][ 'and'] = [f for f in url_filters] if 'filters' in query_file: if 'and' not in query_body['query']['filtered']['filter']: query_body['query']['filtered']['filter']['and'] = [] for json_filter in query_file['filters']: query_body['query']['filtered'][ 'filter']['and'].append(json_filter) final_query_dict = dict((k, v) for (k, v) in query_dict.items() if k in ALLOWED_SEARCH_PARAMS) final_query_dict['index'] = self.es_index final_query_dict['body'] = query_body response = self.es.search(**final_query_dict) response['query'] = query_dict return QueryResults(response, pagenum)
def test_default_days_incorrect_range(self): args = MultiDict([('filter_range_date_gte', '2014-6'), ('filter_range_date_lte', '2014-1')]) filter_dsl = filters.filter_dsl_from_multidict(args) assert(filter_dsl[0]['range']['date']['gte'] == '2014-1-1') assert(filter_dsl[0]['range']['date']['lte'] == '2014-6-30')
def test_date_validation_with_days_correct_range(self): args = MultiDict([('filter_range_date_gte', '2014-1-23'), ('filter_range_date_lte', '2014-6-23')]) filter_dsl = filters.filter_dsl_from_multidict(args) assert(filter_dsl[0]['range']['date']['gte'] == '2014-1-23') assert(filter_dsl[0]['range']['date']['lte'] == '2014-6-23')
def search(self, aggregations=None, use_url_arguments=True, **kwargs): query_file = json.loads(file(self.filename).read()) query_dict = query_file['query'] ''' These dict constructors split the kwargs from the template into filter arguments and arguments that can be placed directly into the query body. The dict constructor syntax supports python 2.6, 2.7, and 3.x If python 2.7, use dict comprehension and iteritems() With python 3, use dict comprehension and items() (items() replaces iteritems and is just as fast) ''' filter_args = dict((key, value) for (key, value) in kwargs.items() if key.startswith('filter_')) non_filter_args = dict((key, value) for (key, value) in kwargs.items() if not key.startswith('filter_')) query_dict.update(non_filter_args) pagenum = 1 request = flask.request # Add in filters from the template. new_multidict = MultiDict() # First add the url arguments if requested if use_url_arguments: new_multidict = request.args.copy() # Next add the arguments from the search() function used in the # template for key, value in filter_args.items(): new_multidict.add(key, value) filters = filter_dsl_from_multidict(new_multidict) args_flat = request.args.to_dict(flat=True) query_body = {} if aggregations: aggs_dsl = {} if type(aggregations) is str: aggregations = [aggregations] # so we can treat it as a list for fieldname in aggregations: aggs_dsl[fieldname] = { 'terms': { 'field': fieldname, 'size': 10000 } } query_body['aggs'] = aggs_dsl else: if 'page' in args_flat: args_flat['from_'] = int(query_dict.get( 'size', '10')) * (int(args_flat['page']) - 1) pagenum = int(args_flat['page']) args_flat_filtered = dict([(k, v) for k, v in args_flat.items() if v]) query_dict.update(args_flat_filtered) query_body['query'] = {'filtered': {'filter': {}}} if filters: query_body['query']['filtered']['filter']['and'] = [ f for f in filters ] if 'filters' in query_file: if 'and' not in query_body['query']['filtered']['filter']: query_body['query']['filtered']['filter']['and'] = [] for json_filter in query_file['filters']: query_body['query']['filtered']['filter']['and'].append( json_filter) final_query_dict = dict((k, v) for (k, v) in query_dict.items() if k in ALLOWED_SEARCH_PARAMS) final_query_dict['index'] = self.es_index final_query_dict['body'] = query_body response = self.es.search(**final_query_dict) response['query'] = query_dict return QueryResults(response, pagenum)