def default_home_context_processor(context):
recent_datasets, recent_reuses = search.multiquery(
search.SearchQuery(Dataset, sort='-created', page_size=12),
search.SearchQuery(Reuse, sort='-created', page_size=12),
)
context.update(recent_datasets=recent_datasets,
recent_reuses=recent_reuses)
return context
def render_search():
params = multi_to_dict(request.args)
params['facets'] = True
datasets, organizations, reuses, users = search.multiquery(
search.SearchQuery(Dataset, **params),
search.SearchQuery(Organization, **params),
search.SearchQuery(Reuse, **params),
search.SearchQuery(User, **params),
)
return theme.render('search.html',
datasets=datasets,
organizations=organizations,
reuses=reuses,
users=users)
def render_search():
params = multi_to_dict(request.args)
params['facets'] = True
# We only fetch relevant data for the given filter.
if 'tag' in params:
search_queries = [
search.SearchQuery(Dataset, **params),
search.SearchQuery(Reuse, **params)
]
results_labels = ['datasets', 'reuses']
elif 'badge' in params:
search_queries = [
search.SearchQuery(Dataset, **params),
search.SearchQuery(Organization, **params)
]
results_labels = ['datasets', 'organizations']
else:
search_queries = [
search.SearchQuery(Dataset, **params),
search.SearchQuery(Reuse, **params),
search.SearchQuery(Organization, **params),
search.SearchQuery(User, **params)
]
results_labels = ['datasets', 'reuses', 'organizations', 'users']
results = search.multiquery(*search_queries)
context = dict(zip(results_labels, results))
context['territories'] = check_for_territories(params.get('q'))
return theme.render('search.html', **context)
def test_facet_filter_extras(self):
search_query = search.SearchQuery(
FakeSearch, **{
'q': 'test',
'extra.key': 'value'
})
expectations = [
{
'multi_match': {
'query': 'test',
'analyzer': search.i18n_analyzer,
'type': 'cross_fields',
'fields': ['title^2', 'description']
}
},
{
'term': {
'extras.key': 'value'
}
},
]
query = search_query.get_query()
self.assertEqual(len(query['bool']['must']), len(expectations))
for expected in expectations:
self.assertIn(expected, query['bool']['must'])
def test_iterate(self):
with self.autoindex():
objects = [FakeFactory() for _ in range(5)]
query = search.SearchQuery(FakeSearch)
for idx, obj in enumerate(query.iter(), 1):
self.assertIsInstance(obj, Fake)
self.assertEqual(idx, len(objects))
def test_empty_search(self):
'''An empty query should match all documents'''
search_query = search.SearchQuery(FakeSearch)
body = search_query.get_body()
self.assertEqual(body['query'], {'match_all': {}})
self.assertEqual(body['facets'], {})
self.assertEqual(body['sort'], [])
def test_paginated_search(self):
'''Search should handle pagination'''
search_query = search.SearchQuery(FakeSearch, page=3, page_size=10)
body = search_query.get_body()
self.assertIn('from', body)
self.assertEqual(body['from'], 20)
self.assertIn('size', body)
self.assertEqual(body['size'], 10)
def test_no_pagination_in_query(self):
'''Search results should be paginated even if not asked'''
query = search.SearchQuery(FakeSearch)
result = search.SearchResult(query, {})
self.assertEqual(result.page, 1),
self.assertEqual(result.page_size, search.DEFAULT_PAGE_SIZE)
self.assertEqual(result.pages, 0)
def test_pagination_empty(self):
'''Search results should be paginated even if empty'''
query = search.SearchQuery(FakeSearch, page=2, page_size=3)
result = search.SearchResult(query, {})
self.assertEqual(result.page, 1),
self.assertEqual(result.page_size, 3)
self.assertEqual(result.pages, 0)
def test_pagination(self):
'''Search results should be paginated'''
kwargs = {'page': 2, 'page_size': 3}
query = search.SearchQuery(FakeSearch, **kwargs)
result = search.SearchResult(query, es_factory(nb=3, total=11))
self.assertEqual(result.page, 2),
self.assertEqual(result.page_size, 3)
self.assertEqual(result.pages, 4)
def test_selected_facets(self):
selected_facets = ['tag', 'other']
search_query = search.SearchQuery(FakeSearch, facets=selected_facets)
facets = search_query.get_facets()
self.assertEqual(len(facets), len(selected_facets))
for key in FakeSearch.facets.keys():
if key in selected_facets:
self.assertIn(key, facets.keys())
else:
self.assertNotIn(key, facets.keys())
def test_no_failures(self):
'''Search result should not fail on missing properties'''
query = search.SearchQuery(FakeSearch)
result = search.SearchResult(query, {})
self.assertEqual(result.total, 0)
self.assertEqual(result.max_score, 0)
ids = result.get_ids()
self.assertEqual(len(ids), 0)
def test_properties(self):
'''Search result should map some properties for easy access'''
response = es_factory(nb=10, total=42)
max_score = response['hits']['max_score']
query = search.SearchQuery(FakeSearch)
result = search.SearchResult(query, response)
self.assertEqual(result.total, 42)
self.assertEqual(result.max_score, max_score)
ids = result.get_ids()
self.assertEqual(len(ids), 10)
def test_custom_function_scoring(self):
'''Search should handle field boosting by function'''
class FakeBoostedSearch(FakeSearch):
boosters = [search.FunctionBooster('doc["field"].value * 2')]
query = search.SearchQuery(FakeBoostedSearch)
body = query.get_body()
# Query should be wrapped in function_score
self.assertEqual(body['query']['function_score']['functions'][0], {
'script_score': {
'script': 'doc["field"].value * 2'
},
})
def test_multi_sorted_search(self):
'''Search should sort'''
search_query = search.SearchQuery(FakeSearch,
sort=['-title', 'description'])
body = search_query.get_body()
self.assertEqual(body['sort'], [
{
'title.raw': 'desc'
},
{
'description.raw': 'asc'
},
])
def test_facet_filter_multi(self):
search_query = search.SearchQuery(FakeSearch,
q='test',
tag=['value-1', 'value-2'],
other='value',
range='3-8',
daterange='2013-01-07-2014-06-07')
expectations = [
{
'multi_match': {
'query': 'test',
'analyzer': search.i18n_analyzer,
'type': 'cross_fields',
'fields': ['title^2', 'description']
}
},
{
'term': {
'tags': 'value-1'
}
},
{
'term': {
'tags': 'value-2'
}
},
{
'term': {
'other': 'value'
}
},
{
'range': {
'a_num_field': {
'gte': 3,
'lte': 8,
}
}
},
{
'range': {
'a_daterange_field': {
'lte': '2014-06-07',
'gte': '2013-01-07',
},
}
},
]
query = search_query.get_query()
for expected in expectations:
self.assertIn(expected, query['bool']['must'])
def test_decay_function_scoring_with_options(self):
'''Search should handle field decay with options'''
class FakeBoostedSearch(FakeSearch):
boosters = [
search.GaussDecay('a_num_field', 10, 20, offset=5, decay=0.5),
search.ExpDecay('another_field',
20,
scale=30,
offset=5,
decay=0.5),
search.LinearDecay('last_field', 30, 40, offset=5, decay=0.5),
]
query = search.SearchQuery(FakeBoostedSearch)
body = query.get_body()
functions = body['query']['function_score']['functions']
# Query should be wrapped in a gaus decay function
self.assertEqual(
functions[0], {
'gauss': {
'a_num_field': {
'origin': 10,
'scale': 20,
'offset': 5,
'decay': 0.5,
}
},
})
self.assertEqual(
functions[1], {
'exp': {
'another_field': {
'origin': 20,
'scale': 30,
'offset': 5,
'decay': 0.5,
}
},
})
self.assertEqual(
functions[2], {
'linear': {
'last_field': {
'origin': 30,
'scale': 40,
'offset': 5,
'decay': 0.5
}
},
})
def test_simple_excluding_query(self):
'''A simple query should negate a simple term in query_string'''
search_query = search.SearchQuery(FakeSearch, q='-test')
expected = {
'bool': {
'must_not': [{
'multi_match': {
'query': 'test',
'analyzer': search.i18n_analyzer,
'type': 'cross_fields',
'fields': ['title^2', 'description']
}
}]
}
}
self.assertEqual(search_query.get_query(), expected)
def test_simple_query_flatten(self):
'''A simple query should use query_string with specified fields and should flatten'''
search_query = search.SearchQuery(FakeSearch, q='test')
expected = {
'bool': {
'must': [{
'multi_match': {
'query': 'test',
'analyzer': search.i18n_analyzer,
'type': 'cross_fields',
'fields': ['title^2', 'description']
}
}]
}
}
self.assertEqual(search_query.get_query(), expected)
def test_to_url_with_none(self):
kwargs = {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': 2,
}
search_query = search.SearchQuery(FakeSearch, **kwargs)
with self.app.test_request_context('/an_url'):
url = search_query.to_url(tag=None, other='value', replace=True)
parsed_url = url_parse(url)
qs = url_decode(parsed_url.query)
self.assertEqual(parsed_url.path, '/an_url')
self.assertEqual(multi_to_dict(qs), {
'q': 'test',
'other': 'value',
})
def test_to_url_with_specified_url(self):
kwargs = {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': 2,
}
search_query = search.SearchQuery(FakeSearch, **kwargs)
with self.app.test_request_context('/an_url'):
url = search_query.to_url('/another_url')
parsed_url = url_parse(url)
qs = url_decode(parsed_url.query)
self.assertEqual(parsed_url.path, '/another_url')
self.assertEqual(multi_to_dict(qs), {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': '2',
})
def test_simple_query_fuzzy(self):
'''A simple query should use query_string with specified fields'''
search_query = search.SearchQuery(FuzzySearch, q='test')
expected = {
'bool': {
'must': [{
'multi_match': {
'query': 'test',
'analyzer': search.i18n_analyzer,
'type': 'cross_fields',
'fields': ['title^2', 'description'],
'fuzziness': 'AUTO',
'prefix_length': 2,
}
}]
}
}
self.assertEqual(search_query.get_query(), expected)
def test_custom_scoring(self):
'''Search should handle field boosting'''
class FakeBoostedSearch(FakeSearch):
boosters = [search.BoolBooster('some_bool_field', 1.1)]
query = search.SearchQuery(FakeBoostedSearch)
body = query.get_body()
# Query should be wrapped in function_score
self.assertIn('function_score', body['query'])
self.assertIn('query', body['query']['function_score'])
self.assertIn('functions', body['query']['function_score'])
self.assertEqual(body['query']['function_score']['functions'][0], {
'filter': {
'term': {
'some_bool_field': True
}
},
'boost_factor': 1.1,
})
def test_default_type(self):
'''Default analyzer is overridable'''
class FakeAnalyzerSearch(FakeSearch):
match_type = 'most_fields'
search_query = search.SearchQuery(FakeAnalyzerSearch, q='test')
expected = {
'bool': {
'must': [{
'multi_match': {
'query': 'test',
'analyzer': search.i18n_analyzer,
'type': 'most_fields',
'fields': ['title^2', 'description']
}
}]
}
}
self.assertEqual(search_query.get_query(), expected)
def test_query_with_both_including_and_excluding_terms(self):
'''A simple query should detect negation on each term in query_string'''
search_query = search.SearchQuery(FakeSearch, q='test -negated')
expected = {
'bool': {
'must': [{
'multi_match': {
'query': 'test',
'analyzer': search.i18n_analyzer,
'type': 'cross_fields',
'fields': ['title^2', 'description']
}
}],
'must_not': [{
'multi_match': {
'query': 'negated',
'analyzer': search.i18n_analyzer,
'type': 'cross_fields',
'fields': ['title^2', 'description']
}
}]
}
}
self.assertEqual(search_query.get_query(), expected)
def test_decay_function_scoring_with_callables(self):
'''Search should handle field decay with options'''
get_dot5 = lambda: 0.5 # noqa
get_5 = lambda: 5 # noqa
get_10 = lambda: 10 # noqa
get_20 = lambda: 20 # noqa
get_30 = lambda: 30 # noqa
get_40 = lambda: 40 # noqa
class FakeBoostedSearch(FakeSearch):
boosters = [
search.GaussDecay('a_num_field',
get_10,
get_20,
offset=get_5,
decay=get_dot5),
search.ExpDecay('another_field',
get_20,
scale=get_30,
offset=get_5,
decay=get_dot5),
search.LinearDecay('last_field',
get_30,
get_40,
offset=get_5,
decay=get_dot5),
]
query = search.SearchQuery(FakeBoostedSearch)
body = query.get_body()
functions = body['query']['function_score']['functions']
# Query should be wrapped in a gaus decay function
self.assertEqual(
functions[0], {
'gauss': {
'a_num_field': {
'origin': 10,
'scale': 20,
'offset': 5,
'decay': 0.5,
}
},
})
self.assertEqual(
functions[1], {
'exp': {
'another_field': {
'origin': 20,
'scale': 30,
'offset': 5,
'decay': 0.5,
}
},
})
self.assertEqual(
functions[2], {
'linear': {
'last_field': {
'origin': 30,
'scale': 40,
'offset': 5,
'decay': 0.5
}
},
})
def test_sorted_search_asc(self):
'''Search should sort by field in ascending order'''
search_query = search.SearchQuery(FakeSearch, sort='title')
body = search_query.get_body()
self.assertEqual(body['sort'], [{'title.raw': 'asc'}])
def test_iterate_empty(self):
with self.autoindex():
[FakeFactory() for _ in range(5)]
query = search.SearchQuery(FakeSearch, tag='not-found')
self.assertEqual(len(list(query.iter())), 0)
def test_facets(self):
search_query = search.SearchQuery(FakeSearch, facets=True)
facets = search_query.get_facets()
self.assertEqual(len(facets), len(FakeSearch.facets))
for key in FakeSearch.facets.keys():
self.assertIn(key, facets.keys())
