def search_for_type(
search_term, document_type, model, schema, adapt_schema, limit, lang):
# search in all title* (title_en, title_fr, ...), summary* and
# description* fields. "boost" title fields and summary fields.
search_query = MultiMatch(
query=search_term,
fields=['title*^3', 'summary*^2', 'description*']
)
# filter on the document_type
type_query = Term(doc_type=document_type)
search = create_search().\
query(search_query).\
filter(type_query).\
fields([]).\
extra(from_=0, size=limit)
# only request the document ids from ES
response = search.execute()
document_ids = [int(doc.meta.id) for doc in response]
# then load the documents for the returned ids
documents = get_documents(document_ids, model, lang)
count = len(documents)
total = response.hits.total
return {
'count': count,
'total': total,
'documents': [
to_json_dict(doc, adapt_schema(schema, doc)) for doc in documents
]
}
def test_build_query_limit_offset(self):
params = {"q": "search word"}
meta_params = {"limit": 20, "offset": 40}
query = build_query(params, meta_params, "w")
expected_query = (
create_search("w").query(get_text_query_on_title("search word")).fields([]).extra(from_=40, size=20)
)
self.assertQueryEqual(query, expected_query)
def test_build_query_limit_offset(self):
params = {'q': 'search word'}
meta_params = {'limit': 20, 'offset': 40}
query = build_query(params, meta_params, 'w')
expected_query = create_search('w'). \
query(get_text_query_on_title('search word')). \
fields([]).\
extra(from_=40, size=20)
self.assertQueryEqual(query, expected_query)
def test_build_query_sort_outing(self):
params = {'act': 'skitouring'}
meta_params = {'limit': 20, 'offset': 40}
query = build_query(params, meta_params, 'o')
expected_query = create_search('o'). \
filter(Term(activities='skitouring')).\
fields([]).\
sort({'date_end': {'order': 'desc'}}, {'id': {'order': 'desc'}}).\
extra(from_=40, size=20)
self.assertQueryEqual(query, expected_query)
def test_build_query_sort_outing(self):
params = {"act": "skitouring"}
meta_params = {"limit": 20, "offset": 40}
query = build_query(params, meta_params, "o")
expected_query = (
create_search("o")
.filter(Term(activities="skitouring"))
.fields([])
.sort({"date_end": {"order": "desc"}}, {"id": {"order": "desc"}})
.extra(from_=40, size=20)
)
self.assertQueryEqual(query, expected_query)
def build_query(url_params, meta_params, doc_type):
"""Creates an ElasticSearch query from query parameters.
"""
search_term = url_params.get('q', '').strip()
limit = meta_params.get('limit')
offset = meta_params.get('offset')
search = create_search(doc_type)
if search_term:
search = search.query(
get_text_query_on_title(search_term, meta_params.get('lang')))
search_model = search_documents[doc_type]
for param in url_params:
if param in reserved_query_fields:
continue
filter = create_filter(param, url_params.get(param), search_model)
if filter:
search = search.filter(filter)
search = search.\
fields([]).\
extra(from_=offset, size=limit)
if url_params.get('bbox'):
bbox_filter = create_bbox_filter(url_params.get('bbox'))
if bbox_filter:
search = search.filter(bbox_filter)
if url_params.get('sort'):
keys_to_sort = url_params.get('sort').split(',')
search = search.sort(*keys_to_sort)
elif not search_term:
# if a search term is given, the documents are sorted by a relevance
# score. if not explicitly sort by id/date.
if doc_type == OUTING_TYPE:
search = search.sort({'date_end': {
'order': 'desc'
}}, {'id': {
'order': 'desc'
}})
elif doc_type == XREPORT_TYPE:
search = search.sort({'date': {
'order': 'desc'
}}, {'id': {
'order': 'desc'
}})
else:
search = search.sort({'id': {'order': 'desc'}})
log.debug('Search filter: {}'.format(search.to_dict()))
return search
def test_build_query(self):
params = {"q": "search word", "walt": "1500", "a": "1234,4567", "l": "fr"}
meta_params = {"limit": 10, "offset": 0}
query = build_query(params, meta_params, "w")
expected_query = (
create_search("w")
.query(get_text_query_on_title("search word"))
.filter(Term(available_locales="fr"))
.filter(Terms(areas=[1234, 4567]))
.filter(Range(elevation={"gte": 1500}))
.fields([])
.extra(from_=0, size=10)
)
self.assertQueryEqual(query, expected_query)
def test_build_query_limit_offset(self):
params = {
'q': 'search word'
}
meta_params = {
'limit': 20,
'offset': 40
}
query = build_query(params, meta_params, 'w')
expected_query = create_search('w'). \
query(get_text_query('search word')). \
fields([]).\
extra(from_=40, size=20)
self.assertQueryEqual(query, expected_query)
def test_build_query_sort_outing(self):
params = {
'act': 'skitouring'
}
meta_params = {
'limit': 20,
'offset': 40
}
query = build_query(params, meta_params, 'o')
expected_query = create_search('o'). \
filter(Term(activities='skitouring')).\
fields([]).\
sort({'date_end': {'order': 'desc'}}, {'id': {'order': 'desc'}}).\
extra(from_=40, size=20)
self.assertQueryEqual(query, expected_query)
def test_build_query(self):
params = {
'q': 'search word',
'walt': '1500',
'a': '1234,4567',
'l': 'fr'
}
meta_params = {'limit': 10, 'offset': 0}
query = build_query(params, meta_params, 'w')
expected_query = create_search('w'). \
query(get_text_query_on_title('search word')). \
filter(Term(available_locales='fr')).\
filter(Terms(areas=[1234, 4567])). \
filter(Range(elevation={'gte': 1500})). \
fields([]).\
extra(from_=0, size=10)
self.assertQueryEqual(query, expected_query)
def test_build_query_bbox(self):
params = {"q": "search word", "walt": "1500", "bbox": "699398,5785365,699498,5785465"}
meta_params = {"limit": 10, "offset": 0}
query = build_query(params, meta_params, "w")
expected_query = (
create_search("w")
.query(get_text_query_on_title("search word"))
.filter(Range(elevation={"gte": 1500}))
.filter(
GeoBoundingBox(
geom={"left": 6.28279913, "bottom": 46.03129072, "right": 6.28369744, "top": 46.03191439},
type="indexed",
)
)
.fields([])
.extra(from_=0, size=10)
)
self.assertQueryEqual(query, expected_query)
def test_build_query_bbox(self):
params = {
'q': 'search word',
'walt': '1500',
'bbox': '699398,5785365,699498,5785465'
}
meta_params = {'limit': 10, 'offset': 0}
query = build_query(params, meta_params, 'w')
expected_query = create_search('w'). \
query(get_text_query_on_title('search word')). \
filter(Range(elevation={'gte': 1500})). \
filter(GeoBoundingBox(
geom={
'left': 6.28279913, 'bottom': 46.03129072,
'right': 6.28369744, 'top': 46.03191439},
type='indexed')).\
fields([]).\
extra(from_=0, size=10)
self.assertQueryEqual(query, expected_query)
def test_build_query(self):
params = {
'q': 'search word',
'walt': '1500',
'a': '1234,4567',
'l': 'fr'
}
meta_params = {
'limit': 10,
'offset': 0
}
query = build_query(params, meta_params, 'w')
expected_query = create_search('w'). \
query(get_text_query('search word')). \
filter(Term(available_locales='fr')).\
filter(Terms(areas=[1234, 4567])). \
filter(Range(elevation={'gte': 1500})). \
fields([]).\
extra(from_=0, size=10)
self.assertQueryEqual(query, expected_query)
def test_build_query_bbox(self):
params = {
'q': 'search word',
'walt': '1500',
'bbox': '699398,5785365,699498,5785465'
}
meta_params = {
'limit': 10,
'offset': 0
}
query = build_query(params, meta_params, 'w')
expected_query = create_search('w'). \
query(get_text_query('search word')). \
filter(Range(elevation={'gte': 1500})). \
filter(GeoBoundingBox(
geom={
'left': 6.28279913, 'bottom': 46.03129072,
'right': 6.28369744, 'top': 46.03191439},
type='indexed')).\
fields([]).\
extra(from_=0, size=10)
self.assertQueryEqual(query, expected_query)
def build_query(url_params, meta_params, doc_type):
"""Creates an ElasticSearch query from query parameters.
"""
search_term = url_params.get('q', '').strip()
limit = meta_params.get('limit')
offset = meta_params.get('offset')
search = create_search(doc_type)
if search_term:
search = search.query(get_text_query(search_term))
search_model = search_documents[doc_type]
for param in url_params:
if param in reserved_query_fields:
continue
filter = create_filter(param, url_params.get(param), search_model)
if filter:
search = search.filter(filter)
search = search.\
fields([]).\
extra(from_=offset, size=limit)
if url_params.get('bbox'):
bbox_filter = create_bbox_filter(url_params.get('bbox'))
if bbox_filter:
search = search.filter(bbox_filter)
if not search_term:
# if a search term is given, the documents are sorted by a relevance
# score. if not explicitly sort by id/date.
if doc_type == OUTING_TYPE:
search = search.sort(
{'date_end': {'order': 'desc'}}, {'id': {'order': 'desc'}})
else:
search = search.sort({'id': {'order': 'desc'}})
return search
def do_multi_search_for_types(search_types, search_term, limit, lang):
""" Executes a multi-search for all document types in a single request
and returns a list of tuples (document_ids, total) containing the results
for each type.
"""
multi_search = MultiSearch(index=elasticsearch_config['index'])
for search_type in search_types:
(_, get_documents_config) = search_type
search = create_search(get_documents_config.document_type).\
query(get_text_query_on_title(search_term, lang)).\
fields([]).\
extra(from_=0, size=limit)
multi_search = multi_search.add(search)
responses = multi_search.execute()
results_for_type = []
for response in responses:
# only requesting the document ids from ES
document_ids = [int(doc.meta.id) for doc in response]
total = response.hits.total
results_for_type.append((document_ids, total))
return results_for_type
def do_multi_search_for_types(search_types, search_term, limit, lang):
""" Executes a multi-search for all document types in a single request
and returns a list of tuples (document_ids, total) containing the results
for each type.
"""
multi_search = MultiSearch(index=elasticsearch_config['index'])
for search_type in search_types:
(_, get_documents_config) = search_type
search = create_search(get_documents_config.document_type).\
query(get_text_query_on_title(search_term, lang)).\
fields([]).\
extra(from_=0, size=limit)
multi_search = multi_search.add(search)
responses = multi_search.execute()
results_for_type = []
for response in responses:
# only requesting the document ids from ES
document_ids = [int(doc.meta.id) for doc in response]
total = response.hits.total
results_for_type.append((document_ids, total))
return results_for_type