def test_simple_query(self):
query_text = 'the query'
query = RawTextQuery(query_text, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), query_text)
self.assertEquals(len(query.query_parts), 1)
self.assertEquals(len(query.languages), 0)
self.assertFalse(query.specific)
def test_simple_query(self):
query_text = 'the query'
query = RawTextQuery(query_text, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), query_text)
self.assertEquals(len(query.query_parts), 1)
self.assertEquals(len(query.languages), 0)
self.assertFalse(query.specific)
def test_timeout_above1000(self):
query_text = '<3500 the query'
query = RawTextQuery(query_text, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), query_text)
self.assertEquals(len(query.query_parts), 3)
self.assertEquals(query.timeout_limit, 3.5)
self.assertFalse(query.specific)
def autocompleter():
"""Return autocompleter results"""
# set blocked engines
disabled_engines = request.preferences.engines.get_disabled()
# parse query
if PY3:
raw_text_query = RawTextQuery(request.form.get('q', b''),
disabled_engines)
else:
raw_text_query = RawTextQuery(
request.form.get('q', u'').encode('utf-8'), disabled_engines)
raw_text_query.parse_query()
# check if search query is set
if not raw_text_query.getSearchQuery():
return '', 400
# run autocompleter
completer = autocomplete_backends.get(
request.preferences.get_value('autocomplete'))
# parse searx specific autocompleter results like !bang
raw_results = searx_bang(raw_text_query)
# normal autocompletion results only appear if no inner results returned
# and there is a query part besides the engine and language bangs
if len(raw_results) == 0 and completer and (
len(raw_text_query.query_parts) > 1 or
(len(raw_text_query.languages) == 0 and not raw_text_query.specific)):
# get language from cookie
language = request.preferences.get_value('language')
if not language or language == 'all':
language = 'en'
else:
language = language.split('-')[0]
# run autocompletion
raw_results.extend(completer(raw_text_query.getSearchQuery(),
language))
# parse results (write :language and !engine back to result string)
results = []
for result in raw_results:
raw_text_query.changeSearchQuery(result)
# add parsed result
results.append(raw_text_query.getFullQuery())
# return autocompleter results
if request.form.get('format') == 'x-suggestions':
return Response(json.dumps([raw_text_query.query, results]),
mimetype='application/json')
return Response(json.dumps(results), mimetype='application/json')
def test_invalid_language_code(self):
language = 'not_a_language'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), full_query)
self.assertEquals(len(query.query_parts), 1)
self.assertEquals(len(query.languages), 0)
self.assertFalse(query.specific)
def test_unlisted_language_code(self):
language = 'all'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), full_query)
self.assertEquals(len(query.query_parts), 3)
self.assertIn('all', query.languages)
self.assertFalse(query.specific)
def test_invalid_language_code(self):
language = 'not_a_language'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), full_query)
self.assertEquals(len(query.query_parts), 1)
self.assertEquals(len(query.languages), 0)
self.assertFalse(query.specific)
def test_unlisted_language_code(self):
language = 'all'
query_text = 'the query'
full_query = ':' + language + ' ' + query_text
query = RawTextQuery(full_query, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), full_query)
self.assertEquals(len(query.query_parts), 3)
self.assertIn('all', query.languages)
self.assertFalse(query.specific)
def test_timeout_invalid(self):
# invalid number: it is not bang but it is part of the query
query_text = '<xxx the query'
query = RawTextQuery(query_text, [])
query.parse_query()
self.assertEquals(query.getFullQuery(), query_text)
self.assertEquals(len(query.query_parts), 1)
self.assertEquals(query.query_parts[0], query_text)
self.assertEquals(query.timeout_limit, None)
self.assertFalse(query.specific)
def autocompleter():
"""Return autocompleter results"""
# set blocked engines
disabled_engines = request.preferences.engines.get_disabled()
# parse query
if PY3:
raw_text_query = RawTextQuery(request.form.get('q', b''), disabled_engines)
else:
raw_text_query = RawTextQuery(request.form.get('q', u'').encode('utf-8'), disabled_engines)
raw_text_query.parse_query()
# check if search query is set
if not raw_text_query.getSearchQuery():
return '', 400
# run autocompleter
completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))
# parse searx specific autocompleter results like !bang
raw_results = searx_bang(raw_text_query)
# normal autocompletion results only appear if max 3 inner results returned
if len(raw_results) <= 3 and completer:
# get language from cookie
language = request.preferences.get_value('language')
if not language or language == 'all':
language = 'en'
else:
language = language.split('-')[0]
# run autocompletion
raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
# parse results (write :language and !engine back to result string)
results = []
for result in raw_results:
raw_text_query.changeSearchQuery(result)
# add parsed result
results.append(raw_text_query.getFullQuery())
# return autocompleter results
if request.form.get('format') == 'x-suggestions':
return Response(json.dumps([raw_text_query.query, results]),
mimetype='application/json')
return Response(json.dumps(results),
mimetype='application/json')
def autocompleter():
"""Return autocompleter results"""
# set blocked engines
disabled_engines = request.preferences.engines.get_disabled()
# parse query
raw_text_query = RawTextQuery(request.form.get("q", "").encode("utf-8"), disabled_engines)
raw_text_query.parse_query()
# check if search query is set
if not raw_text_query.getSearchQuery():
return "", 400
# run autocompleter
completer = autocomplete_backends.get(request.preferences.get_value("autocomplete"))
# parse searx specific autocompleter results like !bang
raw_results = searx_bang(raw_text_query)
# normal autocompletion results only appear if max 3 inner results returned
if len(raw_results) <= 3 and completer:
# get language from cookie
language = request.preferences.get_value("language")
if not language or language == "all":
language = "en"
else:
language = language.split("_")[0]
# run autocompletion
raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
# parse results (write :language and !engine back to result string)
results = []
for result in raw_results:
raw_text_query.changeSearchQuery(result)
# add parsed result
results.append(raw_text_query.getFullQuery())
# return autocompleter results
if request.form.get("format") == "x-suggestions":
return Response(json.dumps([raw_text_query.query, results]), mimetype="application/json")
return Response(json.dumps(results), mimetype="application/json")
def get_search_query_from_webapp(preferences, form):
# no text for the query ?
if not form.get('q'):
raise SearxParameterException('q', '')
# set blocked engines
disabled_engines = preferences.engines.get_disabled()
# parse query, if tags are set, which change
# the serch engine or search-language
raw_text_query = RawTextQuery(form['q'], disabled_engines)
raw_text_query.parse_query()
# set query
query = raw_text_query.getSearchQuery()
# get and check page number
pageno_param = form.get('pageno', '1')
if not pageno_param.isdigit() or int(pageno_param) < 1:
raise SearxParameterException('pageno', pageno_param)
query_pageno = int(pageno_param)
# get language
# set specific language if set on request, query or preferences
# TODO support search with multible languages
if len(raw_text_query.languages):
query_lang = raw_text_query.languages[-1]
elif 'language' in form:
query_lang = form.get('language')
else:
query_lang = preferences.get_value('language')
# provides backwards compatibility for requests using old language default
if query_lang == 'all':
query_lang = settings['search']['language']
# check language
if not VALID_LANGUAGE_CODE.match(query_lang):
raise SearxParameterException('language', query_lang)
# get safesearch
if 'safesearch' in form:
query_safesearch = form.get('safesearch')
# first check safesearch
if not query_safesearch.isdigit():
raise SearxParameterException('safesearch', query_safesearch)
query_safesearch = int(query_safesearch)
else:
query_safesearch = preferences.get_value('safesearch')
# safesearch : second check
if query_safesearch < 0 or query_safesearch > 2:
raise SearxParameterException('safesearch', query_safesearch)
# get time_range
query_time_range = form.get('time_range')
# check time_range
if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
raise SearxParameterException('time_range', query_time_range)
# query_engines
query_engines = raw_text_query.engines
# query_categories
query_categories = []
# if engines are calculated from query,
# set categories by using that informations
if query_engines and raw_text_query.specific:
query_categories = list(set(engine['category']
for engine in query_engines))
# otherwise, using defined categories to
# calculate which engines should be used
else:
# set categories/engines
load_default_categories = True
for pd_name, pd in form.items():
if pd_name == 'categories':
query_categories.extend(categ for categ in map(unicode.strip, pd.split(',')) if categ in categories)
elif pd_name == 'engines':
pd_engines = [{'category': engines[engine].categories[0],
'name': engine}
for engine in map(unicode.strip, pd.split(',')) if engine in engines]
if pd_engines:
query_engines.extend(pd_engines)
load_default_categories = False
elif pd_name.startswith('category_'):
category = pd_name[9:]
# if category is not found in list, skip
if category not in categories:
continue
if pd != 'off':
# add category to list
query_categories.append(category)
elif category in query_categories:
# remove category from list if property is set to 'off'
query_categories.remove(category)
if not load_default_categories:
if not query_categories:
query_categories = list(set(engine['category']
for engine in query_engines))
else:
# if no category is specified for this search,
# using user-defined default-configuration which
# (is stored in cookie)
if not query_categories:
cookie_categories = preferences.get_value('categories')
for ccateg in cookie_categories:
if ccateg in categories:
query_categories.append(ccateg)
# if still no category is specified, using general
# as default-category
if not query_categories:
query_categories = ['general']
# using all engines for that search, which are
# declared under the specific categories
for categ in query_categories:
query_engines.extend({'category': categ,
'name': engine.name}
for engine in categories[categ]
if (engine.name, categ) not in disabled_engines)
return SearchQuery(query, query_engines, query_categories,
query_lang, query_safesearch, query_pageno, query_time_range)
def get_search_query_from_webapp(preferences, form):
query = None
query_engines = []
query_categories = []
query_paging = False
query_pageno = 1
query_lang = 'all'
query_time_range = None
# set blocked engines
disabled_engines = preferences.engines.get_disabled()
# set specific language if set
query_lang = preferences.get_value('language')
# safesearch
query_safesearch = preferences.get_value('safesearch')
# TODO better exceptions
if not form.get('q'):
raise Exception('noquery')
# set pagenumber
pageno_param = form.get('pageno', '1')
if not pageno_param.isdigit() or int(pageno_param) < 1:
pageno_param = 1
query_pageno = int(pageno_param)
# parse query, if tags are set, which change
# the serch engine or search-language
raw_text_query = RawTextQuery(form['q'], disabled_engines)
raw_text_query.parse_query()
# set query
query = raw_text_query.getSearchQuery()
# get last selected language in query, if possible
# TODO support search with multible languages
if len(raw_text_query.languages):
query_lang = raw_text_query.languages[-1]
query_time_range = form.get('time_range')
query_engines = raw_text_query.engines
# if engines are calculated from query,
# set categories by using that informations
if query_engines and raw_text_query.specific:
query_categories = list(set(engine['category']
for engine in query_engines))
# otherwise, using defined categories to
# calculate which engines should be used
else:
# set categories/engines
load_default_categories = True
for pd_name, pd in form.items():
if pd_name == 'categories':
query_categories.extend(categ for categ in map(unicode.strip, pd.split(',')) if categ in categories)
elif pd_name == 'engines':
pd_engines = [{'category': engines[engine].categories[0],
'name': engine}
for engine in map(unicode.strip, pd.split(',')) if engine in engines]
if pd_engines:
query_engines.extend(pd_engines)
load_default_categories = False
elif pd_name.startswith('category_'):
category = pd_name[9:]
# if category is not found in list, skip
if category not in categories:
continue
if pd != 'off':
# add category to list
query_categories.append(category)
elif category in query_categories:
# remove category from list if property is set to 'off'
query_categories.remove(category)
if not load_default_categories:
if not query_categories:
query_categories = list(set(engine['category']
for engine in engines))
else:
# if no category is specified for this search,
# using user-defined default-configuration which
# (is stored in cookie)
if not query_categories:
cookie_categories = preferences.get_value('categories')
for ccateg in cookie_categories:
if ccateg in categories:
query_categories.append(ccateg)
# if still no category is specified, using general
# as default-category
if not query_categories:
query_categories = ['general']
# using all engines for that search, which are
# declared under the specific categories
for categ in query_categories:
query_engines.extend({'category': categ,
'name': engine.name}
for engine in categories[categ]
if (engine.name, categ) not in disabled_engines)
return SearchQuery(query, query_engines, query_categories,
query_lang, query_safesearch, query_pageno, query_time_range)
def get_search_query_from_webapp(preferences, form):
query = None
query_engines = []
query_categories = []
query_paging = False
query_pageno = 1
query_lang = 'all'
query_time_range = None
# set blocked engines
disabled_engines = preferences.engines.get_disabled()
# set specific language if set
query_lang = preferences.get_value('language')
# safesearch
query_safesearch = preferences.get_value('safesearch')
# TODO better exceptions
if not form.get('q'):
raise Exception('noquery')
# set pagenumber
pageno_param = form.get('pageno', '1')
if not pageno_param.isdigit() or int(pageno_param) < 1:
pageno_param = 1
query_pageno = int(pageno_param)
# parse query, if tags are set, which change
# the serch engine or search-language
raw_text_query = RawTextQuery(form['q'], disabled_engines)
raw_text_query.parse_query()
# set query
query = raw_text_query.getSearchQuery()
# get last selected language in query, if possible
# TODO support search with multible languages
if len(raw_text_query.languages):
query_lang = raw_text_query.languages[-1]
query_time_range = form.get('time_range')
query_engines = raw_text_query.engines
# if engines are calculated from query,
# set categories by using that informations
if query_engines and raw_text_query.specific:
query_categories = list(
set(engine['category'] for engine in query_engines))
# otherwise, using defined categories to
# calculate which engines should be used
else:
# set categories/engines
load_default_categories = True
for pd_name, pd in form.items():
if pd_name == 'categories':
query_categories.extend(
categ for categ in map(unicode.strip, pd.split(','))
if categ in categories)
elif pd_name == 'engines':
pd_engines = [{
'category': engines[engine].categories[0],
'name': engine
} for engine in map(unicode.strip, pd.split(','))
if engine in engines]
if pd_engines:
query_engines.extend(pd_engines)
load_default_categories = False
elif pd_name.startswith('category_'):
category = pd_name[9:]
# if category is not found in list, skip
if category not in categories:
continue
if pd != 'off':
# add category to list
query_categories.append(category)
elif category in query_categories:
# remove category from list if property is set to 'off'
query_categories.remove(category)
if not load_default_categories:
if not query_categories:
query_categories = list(
set(engine['category'] for engine in engines))
else:
# if no category is specified for this search,
# using user-defined default-configuration which
# (is stored in cookie)
if not query_categories:
cookie_categories = preferences.get_value('categories')
for ccateg in cookie_categories:
if ccateg in categories:
query_categories.append(ccateg)
# if still no category is specified, using general
# as default-category
if not query_categories:
query_categories = ['general']
# using all engines for that search, which are
# declared under the specific categories
for categ in query_categories:
query_engines.extend({
'category': categ,
'name': engine.name
} for engine in categories[categ]
if (engine.name,
categ) not in disabled_engines)
return SearchQuery(query, query_engines, query_categories, query_lang,
query_safesearch, query_pageno, query_time_range)
def get_search_query_from_webapp(preferences, form):
# no text for the query ?
if not form.get('q'):
raise SearxParameterException('q', '')
# set blocked engines
disabled_engines = preferences.engines.get_disabled()
# parse query, if tags are set, which change
# the serch engine or search-language
raw_text_query = RawTextQuery(form['q'], disabled_engines)
raw_text_query.parse_query()
# set query
query = raw_text_query.getSearchQuery()
# get and check page number
pageno_param = form.get('pageno', '1')
if not pageno_param.isdigit() or int(pageno_param) < 1:
raise SearxParameterException('pageno', pageno_param)
query_pageno = int(pageno_param)
# get language
# set specific language if set on request, query or preferences
# TODO support search with multible languages
if len(raw_text_query.languages):
query_lang = raw_text_query.languages[-1]
elif 'language' in form:
query_lang = form.get('language')
else:
query_lang = preferences.get_value('language')
# check language
if not VALID_LANGUAGE_CODE.match(query_lang):
raise SearxParameterException('language', query_lang)
# get safesearch
if 'safesearch' in form:
query_safesearch = form.get('safesearch')
# first check safesearch
if not query_safesearch.isdigit():
raise SearxParameterException('safesearch', query_safesearch)
query_safesearch = int(query_safesearch)
else:
query_safesearch = preferences.get_value('safesearch')
# safesearch : second check
if query_safesearch < 0 or query_safesearch > 2:
raise SearxParameterException('safesearch', query_safesearch)
# get time_range
query_time_range = form.get('time_range')
# check time_range
if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
raise SearxParameterException('time_range', query_time_range)
# query_engines
query_engines = raw_text_query.engines
# query_categories
query_categories = []
# if engines are calculated from query,
# set categories by using that informations
if query_engines and raw_text_query.specific:
query_categories = list(set(engine['category']
for engine in query_engines))
# otherwise, using defined categories to
# calculate which engines should be used
else:
# set categories/engines
load_default_categories = True
for pd_name, pd in form.items():
if pd_name == 'categories':
query_categories.extend(categ for categ in map(unicode.strip, pd.split(',')) if categ in categories)
elif pd_name == 'engines':
pd_engines = [{'category': engines[engine].categories[0],
'name': engine}
for engine in map(unicode.strip, pd.split(',')) if engine in engines]
if pd_engines:
query_engines.extend(pd_engines)
load_default_categories = False
elif pd_name.startswith('category_'):
category = pd_name[9:]
# if category is not found in list, skip
if category not in categories:
continue
if pd != 'off':
# add category to list
query_categories.append(category)
elif category in query_categories:
# remove category from list if property is set to 'off'
query_categories.remove(category)
if not load_default_categories:
if not query_categories:
query_categories = list(set(engine['category']
for engine in query_engines))
else:
# if no category is specified for this search,
# using user-defined default-configuration which
# (is stored in cookie)
if not query_categories:
cookie_categories = preferences.get_value('categories')
for ccateg in cookie_categories:
if ccateg in categories:
query_categories.append(ccateg)
# if still no category is specified, using general
# as default-category
if not query_categories:
query_categories = ['general']
# using all engines for that search, which are
# declared under the specific categories
for categ in query_categories:
query_engines.extend({'category': categ,
'name': engine.name}
for engine in categories[categ]
if (engine.name, categ) not in disabled_engines)
return SearchQuery(query, query_engines, query_categories,
query_lang, query_safesearch, query_pageno, query_time_range)
