def autocompleter():
"""Return autocompleter results"""
# set blocked engines
disabled_engines = request.preferences.engines.get_disabled()
# parse query
raw_text_query = RawTextQuery(request.form.get("q", ""), disabled_engines)
# check if search query is set
if not raw_text_query.getQuery():
return "", 400
# run autocompleter
completer = autocomplete_backends.get(
request.preferences.get_value("autocomplete"))
# parse searx specific autocompleter results like !bang
raw_results = searx_bang(raw_text_query)
# normal autocompletion results only appear if no inner results returned
# and there is a query part besides the engine and language bangs
if (len(raw_results) == 0 and completer
and (len(raw_text_query.query_parts) > 1 or
(len(raw_text_query.languages) == 0
and not raw_text_query.specific))):
# get language from cookie
language = request.preferences.get_value("language")
if not language or language == "all":
language = "en"
else:
language = language.split("-")[0]
# run autocompletion
raw_results.extend(completer(raw_text_query.getQuery(), language))
# parse results (write :language and !engine back to result string)
results = []
for result in raw_results:
raw_text_query.changeQuery(result)
# add parsed result
results.append(raw_text_query.getFullQuery())
# return autocompleter results
if request.headers.get("X-Requested-With") == "XMLHttpRequest":
return Response(json.dumps(results), mimetype="application/json")
return Response(
json.dumps([raw_text_query.query, results]),
mimetype="application/x-suggestions+json",
)
def test_bang_autocomplete(self):
initialize(TEST_ENGINES)
query = RawTextQuery('the query !dum', [])
self.assertEqual(query.autocomplete_list, ['!dummy_engine'])
query = RawTextQuery('!dum the query', [])
self.assertEqual(query.autocomplete_list, [])
self.assertEqual(query.getQuery(), '!dum the query')
def autocompleter():
"""Return autocompleter results"""
# run autocompleter
results = []
# set blocked engines
disabled_engines = request.preferences.engines.get_disabled()
# parse query
raw_text_query = RawTextQuery(request.form.get('q', ''), disabled_engines)
# normal autocompletion results only appear if no inner results returned
# and there is a query part
if len(raw_text_query.autocomplete_list) == 0 and len(
raw_text_query.getQuery()) > 0:
# get language from cookie
language = request.preferences.get_value('language')
if not language or language == 'all':
language = 'en'
else:
language = language.split('-')[0]
# run autocompletion
raw_results = search_autocomplete(
request.preferences.get_value('autocomplete'),
raw_text_query.getQuery(), language)
for result in raw_results:
results.append(raw_text_query.changeQuery(result).getFullQuery())
if len(raw_text_query.autocomplete_list) > 0:
for autocomplete_text in raw_text_query.autocomplete_list:
results.append(
raw_text_query.get_autocomplete_full_query(autocomplete_text))
for answers in ask(raw_text_query):
for answer in answers:
results.append(str(answer['answer']))
# return autocompleter results
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return Response(json.dumps(results), mimetype='application/json')
return Response(json.dumps([raw_text_query.query, results]),
mimetype='application/x-suggestions+json')
def test_timeout_invalid(self):
# invalid number: it is not bang but it is part of the query
query_text = '<xxx the query'
query = RawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(len(query.query_parts), 0)
self.assertEqual(query.getQuery(), query_text)
self.assertEqual(query.timeout_limit, None)
self.assertFalse(query.specific)
def test_timeout_autocomplete(self):
# invalid number: it is not bang but it is part of the query
query_text = 'the query <'
query = RawTextQuery(query_text, [])
self.assertEqual(query.getFullQuery(), query_text)
self.assertEqual(len(query.query_parts), 0)
self.assertEqual(query.getQuery(), query_text)
self.assertEqual(query.timeout_limit, None)
self.assertFalse(query.specific)
self.assertEqual(query.autocomplete_list, ['<3', '<850'])
def autocompleter():
"""Return autocompleter results"""
# run autocompleter
results = []
# set blocked engines
disabled_engines = request.preferences.engines.get_disabled()
# parse query
raw_text_query = RawTextQuery(request.form.get('q', ''), disabled_engines)
sug_prefix = raw_text_query.getQuery()
# normal autocompletion results only appear if no inner results returned
# and there is a query part
if len(raw_text_query.autocomplete_list) == 0 and len(sug_prefix) > 0:
# get language from cookie
language = request.preferences.get_value('language')
if not language or language == 'all':
language = 'en'
else:
language = language.split('-')[0]
# run autocompletion
raw_results = search_autocomplete(
request.preferences.get_value('autocomplete'), sug_prefix,
language)
for result in raw_results:
# attention: this loop will change raw_text_query object and this is
# the reason why the sug_prefix was stored before (see above)
results.append(raw_text_query.changeQuery(result).getFullQuery())
if len(raw_text_query.autocomplete_list) > 0:
for autocomplete_text in raw_text_query.autocomplete_list:
results.append(
raw_text_query.get_autocomplete_full_query(autocomplete_text))
for answers in ask(raw_text_query):
for answer in answers:
results.append(str(answer['answer']))
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
# the suggestion request comes from the searx search form
suggestions = json.dumps(results)
mimetype = 'application/json'
else:
# the suggestion request comes from browser's URL bar
suggestions = json.dumps([sug_prefix, results])
mimetype = 'application/x-suggestions+json'
return Response(suggestions, mimetype=mimetype)
def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])\
-> Tuple[SearchQuery, RawTextQuery, List[EngineRef], List[EngineRef]]:
# no text for the query ?
if not form.get('q'):
raise SearxParameterException('q', '')
# set blocked engines
disabled_engines = preferences.engines.get_disabled()
# parse query, if tags are set, which change
# the serch engine or search-language
raw_text_query = RawTextQuery(form['q'], disabled_engines)
# set query
query = raw_text_query.getQuery()
query_pageno = parse_pageno(form)
query_lang = parse_lang(preferences, form, raw_text_query)
query_safesearch = parse_safesearch(preferences, form)
query_time_range = parse_time_range(form)
query_timeout = parse_timeout(form, raw_text_query)
external_bang = raw_text_query.external_bang
if not is_locked(
'categories'
) and raw_text_query.enginerefs and raw_text_query.specific:
# if engines are calculated from query,
# set categories by using that informations
query_engineref_list = raw_text_query.enginerefs
else:
# otherwise, using defined categories to
# calculate which engines should be used
query_engineref_list = parse_generic(preferences, form,
disabled_engines)
query_engineref_list = deduplicate_engineref_list(query_engineref_list)
query_engineref_list, query_engineref_list_unknown, query_engineref_list_notoken =\
validate_engineref_list(query_engineref_list, preferences)
return (SearchQuery(query,
query_engineref_list,
query_lang,
query_safesearch,
query_pageno,
query_time_range,
query_timeout,
external_bang=external_bang), raw_text_query,
query_engineref_list_unknown, query_engineref_list_notoken)
def get_search_query_from_webapp(preferences, form):
# no text for the query ?
if not form.get('q'):
raise SearxParameterException('q', '')
# set blocked engines
disabled_engines = preferences.engines.get_disabled()
# parse query, if tags are set, which change
# the serch engine or search-language
raw_text_query = RawTextQuery(form['q'], disabled_engines)
# set query
query = raw_text_query.getQuery()
# get and check page number
pageno_param = form.get('pageno', '1')
if not pageno_param.isdigit() or int(pageno_param) < 1:
raise SearxParameterException('pageno', pageno_param)
query_pageno = int(pageno_param)
# get language
# set specific language if set on request, query or preferences
# TODO support search with multible languages
if len(raw_text_query.languages):
query_lang = raw_text_query.languages[-1]
elif 'language' in form:
query_lang = form.get('language')
else:
query_lang = preferences.get_value('language')
# check language
if not VALID_LANGUAGE_CODE.match(query_lang):
raise SearxParameterException('language', query_lang)
# get safesearch
if 'safesearch' in form:
query_safesearch = form.get('safesearch')
# first check safesearch
if not query_safesearch.isdigit():
raise SearxParameterException('safesearch', query_safesearch)
query_safesearch = int(query_safesearch)
else:
query_safesearch = preferences.get_value('safesearch')
# safesearch : second check
if query_safesearch < 0 or query_safesearch > 2:
raise SearxParameterException('safesearch', query_safesearch)
# get time_range
query_time_range = form.get('time_range')
# check time_range
if query_time_range not in ('None', None, '', 'day', 'week', 'month',
'year'):
raise SearxParameterException('time_range', query_time_range)
# query_engines
query_engines = raw_text_query.engines
# timeout_limit
query_timeout = raw_text_query.timeout_limit
if query_timeout is None and 'timeout_limit' in form:
raw_time_limit = form.get('timeout_limit')
if raw_time_limit in ['None', '']:
raw_time_limit = None
else:
try:
query_timeout = float(raw_time_limit)
except ValueError:
raise SearxParameterException('timeout_limit', raw_time_limit)
# query_categories
query_categories = []
# if engines are calculated from query,
# set categories by using that informations
if query_engines and raw_text_query.specific:
additional_categories = set()
for engine in query_engines:
if 'from_bang' in engine and engine['from_bang']:
additional_categories.add('none')
else:
additional_categories.add(engine['category'])
query_categories = list(additional_categories)
# otherwise, using defined categories to
# calculate which engines should be used
else:
# set categories/engines
load_default_categories = True
for pd_name, pd in form.items():
if pd_name == 'categories':
query_categories.extend(
categ for categ in map(str.strip, pd.split(','))
if categ in categories)
elif pd_name == 'engines':
pd_engines = [{
'category': engines[engine].categories[0],
'name': engine
} for engine in map(str.strip, pd.split(','))
if engine in engines]
if pd_engines:
query_engines.extend(pd_engines)
load_default_categories = False
elif pd_name.startswith('category_'):
category = pd_name[9:]
# if category is not found in list, skip
if category not in categories:
continue
if pd != 'off':
# add category to list
query_categories.append(category)
elif category in query_categories:
# remove category from list if property is set to 'off'
query_categories.remove(category)
if not load_default_categories:
if not query_categories:
query_categories = list(
set(engine['category'] for engine in query_engines))
else:
# if no category is specified for this search,
# using user-defined default-configuration which
# (is stored in cookie)
if not query_categories:
cookie_categories = preferences.get_value('categories')
for ccateg in cookie_categories:
if ccateg in categories:
query_categories.append(ccateg)
# if still no category is specified, using general
# as default-category
if not query_categories:
query_categories = ['general']
# using all engines for that search, which are
# declared under the specific categories
for categ in query_categories:
query_engines.extend({
'category': categ,
'name': engine.name
} for engine in categories[categ]
if (engine.name,
categ) not in disabled_engines)
query_engines = deduplicate_query_engines(query_engines)
external_bang = raw_text_query.external_bang
return (SearchQuery(query,
query_engines,
query_categories,
query_lang,
query_safesearch,
query_pageno,
query_time_range,
query_timeout,
preferences,
external_bang=external_bang), raw_text_query)
