def index_error(output_format, error_message):
if output_format == 'json':
return Response(json.dumps({'error': error_message}),
mimetype='application/json')
elif output_format == 'csv':
response = Response('', mimetype='application/csv')
cont_disp = 'attachment;Filename=searx.csv'
response.headers.add('Content-Disposition', cont_disp)
return response
elif output_format == 'rss':
response_rss = render(
'opensearch_response_rss.xml',
results=[],
q=request.form['q'] if 'q' in request.form else '',
number_of_results=0,
base_url=get_base_url(),
error_message=error_message,
override_theme='__common__',
)
return Response(response_rss, mimetype='text/xml')
else:
# html
request.errors.append(gettext('search error'))
return render(
'index.html',
selected_categories=get_selected_categories(request.preferences, request.form),
)
def index_error(output_format, error_message):
if output_format == "json":
return Response(json.dumps({"error": error_message}),
mimetype="application/json")
elif output_format == "csv":
response = Response("", mimetype="application/csv")
cont_disp = "attachment;Filename=searx.csv"
response.headers.add("Content-Disposition", cont_disp)
return response
elif output_format == "rss":
response_rss = render(
"opensearch_response_rss.xml",
results=[],
q=request.form["q"] if "q" in request.form else "",
number_of_results=0,
base_url=get_base_url(),
error_message=error_message,
override_theme="__common__",
)
return Response(response_rss, mimetype="text/xml")
else:
# html
request.errors.append(gettext("search error"))
return render(
"index.html",
selected_categories=get_selected_categories(
request.preferences, request.form),
)
def index():
"""Render index page."""
# redirect to search if there's a query in the request
if request.form.get('q'):
query = ('?' + request.query_string.decode()) if request.query_string else ''
return redirect(url_for('search') + query, 308)
return render(
'index.html',
selected_categories=get_selected_categories(request.preferences, request.form),
)
def index():
"""Render index page."""
# UI
advanced_search = request.preferences.get_value("advanced_search")
# redirect to search if there's a query in the request
if request.form.get("q"):
query = ("?" +
request.query_string.decode()) if request.query_string else ""
return redirect(url_for("search") + query, 308)
return render(
"index.html",
selected_categories=get_selected_categories(request.preferences,
request.form),
advanced_search=advanced_search,
)
def preferences():
"""Render preferences page && save user preferences"""
# save preferences
if request.method == 'POST':
resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
try:
request.preferences.parse_form(request.form)
except ValidationException:
request.errors.append(gettext('Invalid settings, please edit your preferences'))
return resp
return request.preferences.save(resp)
# render preferences
image_proxy = request.preferences.get_value('image_proxy')
disabled_engines = request.preferences.engines.get_disabled()
allowed_plugins = request.preferences.plugins.get_enabled()
# stats for preferences page
stats = {}
engines_by_category = {}
for c in categories:
engines_by_category[c] = []
for e in categories[c]:
if not request.preferences.validate_token(e):
continue
stats[e.name] = {'time': None,
'warn_timeout': False,
'warn_time': False}
if e.timeout > settings['outgoing']['request_timeout']:
stats[e.name]['warn_timeout'] = True
stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences)
engines_by_category[c].append(e)
# get first element [0], the engine time,
# and then the second element [1] : the time (the first one is the label)
for engine_stat in get_engines_stats(request.preferences)[0][1]:
stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)
if engine_stat.get('avg') > settings['outgoing']['request_timeout']:
stats[engine_stat.get('name')]['warn_time'] = True
# end of stats
locked_preferences = list()
if 'preferences' in settings and 'lock' in settings['preferences']:
locked_preferences = settings['preferences']['lock']
return render('preferences.html',
selected_categories=get_selected_categories(request.preferences, request.form),
all_categories=_get_ordered_categories(),
locales=settings['locales'],
current_locale=request.preferences.get_value("locale"),
image_proxy=image_proxy,
engines_by_category=engines_by_category,
stats=stats,
answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
disabled_engines=disabled_engines,
autocomplete_backends=autocomplete_backends,
shortcuts={y: x for x, y in engine_shortcuts.items()},
themes=themes,
plugins=plugins,
doi_resolvers=settings['doi_resolvers'],
current_doi_resolver=get_doi_resolver(request.args, request.preferences.get_value('doi_resolver')),
allowed_plugins=allowed_plugins,
theme=get_current_theme_name(),
preferences_url_params=request.preferences.get_as_url_params(),
base_url=get_base_url(),
locked_preferences=locked_preferences,
preferences=True)
def search():
"""Search query in q and return results.
Supported outputs: html, json, csv, rss.
"""
# output_format
output_format = request.form.get('format', 'html')
if output_format not in ['html', 'csv', 'json', 'rss']:
output_format = 'html'
# check if there is query (not None and not an empty string)
if not request.form.get('q'):
if output_format == 'html':
return render(
'index.html',
advanced_search=request.preferences.get_value('advanced_search'),
selected_categories=get_selected_categories(request.preferences, request.form),
)
else:
return index_error(output_format, 'No query'), 400
# search
search_query = None
raw_text_query = None
result_container = None
try:
search_query, raw_text_query, _, _ = get_search_query_from_webapp(request.preferences, request.form)
# search = Search(search_query) # without plugins
search = SearchWithPlugins(search_query, request.user_plugins, request)
result_container = search.search()
except SearxParameterException as e:
logger.exception('search error: SearxParameterException')
return index_error(output_format, e.message), 400
except Exception as e:
logger.exception('search error')
return index_error(output_format, gettext('search error')), 500
# results
results = result_container.get_ordered_results()
number_of_results = result_container.results_number()
if number_of_results < result_container.results_length():
number_of_results = 0
# checkin for a external bang
if result_container.redirect_url:
return redirect(result_container.redirect_url)
# Server-Timing header
request.timings = result_container.get_timings()
# output
for result in results:
if output_format == 'html':
if 'content' in result and result['content']:
result['content'] = highlight_content(escape(result['content'][:1024]), search_query.query)
if 'title' in result and result['title']:
result['title'] = highlight_content(escape(result['title'] or ''), search_query.query)
else:
if result.get('content'):
result['content'] = html_to_text(result['content']).strip()
# removing html content and whitespace duplications
result['title'] = ' '.join(html_to_text(result['title']).strip().split())
if 'url' in result:
result['pretty_url'] = prettify_url(result['url'])
# TODO, check if timezone is calculated right
if 'publishedDate' in result:
try: # test if publishedDate >= 1900 (datetime module bug)
result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S%z')
except ValueError:
result['publishedDate'] = None
else:
if result['publishedDate'].replace(tzinfo=None) >= datetime.now() - timedelta(days=1):
timedifference = datetime.now() - result['publishedDate'].replace(tzinfo=None)
minutes = int((timedifference.seconds / 60) % 60)
hours = int(timedifference.seconds / 60 / 60)
if hours == 0:
result['publishedDate'] = gettext('{minutes} minute(s) ago').format(minutes=minutes)
else:
result['publishedDate'] = gettext('{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes) # noqa
else:
result['publishedDate'] = format_date(result['publishedDate'])
if output_format == 'json':
return Response(json.dumps({'query': search_query.query,
'number_of_results': number_of_results,
'results': results,
'answers': list(result_container.answers),
'corrections': list(result_container.corrections),
'infoboxes': result_container.infoboxes,
'suggestions': list(result_container.suggestions),
'unresponsive_engines': __get_translated_errors(result_container.unresponsive_engines)}, # noqa
default=lambda item: list(item) if isinstance(item, set) else item),
mimetype='application/json')
elif output_format == 'csv':
csv = UnicodeWriter(StringIO())
keys = ('title', 'url', 'content', 'host', 'engine', 'score', 'type')
csv.writerow(keys)
for row in results:
row['host'] = row['parsed_url'].netloc
row['type'] = 'result'
csv.writerow([row.get(key, '') for key in keys])
for a in result_container.answers:
row = {'title': a, 'type': 'answer'}
csv.writerow([row.get(key, '') for key in keys])
for a in result_container.suggestions:
row = {'title': a, 'type': 'suggestion'}
csv.writerow([row.get(key, '') for key in keys])
for a in result_container.corrections:
row = {'title': a, 'type': 'correction'}
csv.writerow([row.get(key, '') for key in keys])
csv.stream.seek(0)
response = Response(csv.stream.read(), mimetype='application/csv')
cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query)
response.headers.add('Content-Disposition', cont_disp)
return response
elif output_format == 'rss':
response_rss = render(
'opensearch_response_rss.xml',
results=results,
answers=result_container.answers,
corrections=result_container.corrections,
suggestions=result_container.suggestions,
q=request.form['q'],
number_of_results=number_of_results,
base_url=get_base_url(),
override_theme='__common__',
)
return Response(response_rss, mimetype='text/xml')
# HTML output format
# suggestions: use RawTextQuery to get the suggestion URLs with the same bang
suggestion_urls = list(map(lambda suggestion: {
'url': raw_text_query.changeQuery(suggestion).getFullQuery(),
'title': suggestion
},
result_container.suggestions))
correction_urls = list(map(lambda correction: {
'url': raw_text_query.changeQuery(correction).getFullQuery(),
'title': correction
},
result_container.corrections))
#
return render(
'results.html',
results=results,
q=request.form['q'],
selected_categories=search_query.categories,
pageno=search_query.pageno,
time_range=search_query.time_range,
number_of_results=format_decimal(number_of_results),
suggestions=suggestion_urls,
answers=result_container.answers,
corrections=correction_urls,
infoboxes=result_container.infoboxes,
paging=result_container.paging,
unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines),
current_language=match_language(search_query.lang,
LANGUAGE_CODES,
fallback=request.preferences.get_value("language")),
base_url=get_base_url(),
theme=get_current_theme_name(),
favicons=global_favicons[themes.index(get_current_theme_name())],
timeout_limit=request.form.get('timeout_limit', None)
)
def preferences():
"""Render preferences page && save user preferences"""
# save preferences
if request.method == "POST":
resp = make_response(
redirect(urljoin(settings["server"]["base_url"],
url_for("index"))))
try:
request.preferences.parse_form(request.form)
except ValidationException:
request.errors.append(
gettext("Invalid settings, please edit your preferences"))
return resp
return request.preferences.save(resp)
# render preferences
image_proxy = request.preferences.get_value("image_proxy")
disabled_engines = request.preferences.engines.get_disabled()
allowed_plugins = request.preferences.plugins.get_enabled()
# stats for preferences page
stats = {}
engines_by_category = {}
for c in categories:
engines_by_category[c] = []
for e in categories[c]:
if not request.preferences.validate_token(e):
continue
stats[e.name] = {
"time": None,
"warn_timeout": False,
"warn_time": False
}
if e.timeout > settings["outgoing"]["request_timeout"]:
stats[e.name]["warn_timeout"] = True
stats[e.name][
"supports_selected_language"] = _is_selected_language_supported(
e, request.preferences)
engines_by_category[c].append(e)
# get first element [0], the engine time,
# and then the second element [1] : the time (the first one is the label)
for engine_stat in get_engines_stats(request.preferences)[0][1]:
stats[engine_stat.get("name")]["time"] = round(engine_stat.get("avg"),
3)
if engine_stat.get("avg") > settings["outgoing"]["request_timeout"]:
stats[engine_stat.get("name")]["warn_time"] = True
# end of stats
locked_preferences = list()
if "preferences" in settings and "lock" in settings["preferences"]:
locked_preferences = settings["preferences"]["lock"]
return render(
"preferences.html",
selected_categories=get_selected_categories(request.preferences,
request.form),
all_categories=_get_ordered_categories(),
locales=settings["locales"],
current_locale=request.preferences.get_value("locale"),
image_proxy=image_proxy,
engines_by_category=engines_by_category,
stats=stats,
answerers=[{
"info": a.self_info(),
"keywords": a.keywords
} for a in answerers],
disabled_engines=disabled_engines,
autocomplete_backends=autocomplete_backends,
shortcuts={y: x
for x, y in engine_shortcuts.items()},
themes=themes,
plugins=plugins,
doi_resolvers=settings["doi_resolvers"],
current_doi_resolver=get_doi_resolver(
request.args, request.preferences.get_value("doi_resolver")),
allowed_plugins=allowed_plugins,
theme=get_current_theme_name(),
preferences_url_params=request.preferences.get_as_url_params(),
base_url=get_base_url(),
locked_preferences=locked_preferences,
preferences=True,
)
def preferences():
"""Render preferences page && save user preferences"""
# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches
# pylint: disable=too-many-statements
# save preferences
if request.method == 'POST':
resp = make_response(redirect(url_for('index', _external=True)))
try:
request.preferences.parse_form(request.form)
except ValidationException:
request.errors.append(
gettext('Invalid settings, please edit your preferences'))
return resp
return request.preferences.save(resp)
# render preferences
image_proxy = request.preferences.get_value('image_proxy') # pylint: disable=redefined-outer-name
disabled_engines = request.preferences.engines.get_disabled()
allowed_plugins = request.preferences.plugins.get_enabled()
# stats for preferences page
filtered_engines = dict(
filter(lambda kv: (kv[0], request.preferences.validate_token(kv[1])),
engines.items()))
engines_by_category = {}
for c in categories:
engines_by_category[c] = [
e for e in categories[c] if e.name in filtered_engines
]
# sort the engines alphabetically since the order in settings.yml is meaningless.
list.sort(engines_by_category[c], key=lambda e: e.name)
# get first element [0], the engine time,
# and then the second element [1] : the time (the first one is the label)
stats = {} # pylint: disable=redefined-outer-name
max_rate95 = 0
for _, e in filtered_engines.items():
h = histogram('engine', e.name, 'time', 'total')
median = round(h.percentage(50), 1) if h.count > 0 else None
rate80 = round(h.percentage(80), 1) if h.count > 0 else None
rate95 = round(h.percentage(95), 1) if h.count > 0 else None
max_rate95 = max(max_rate95, rate95 or 0)
result_count_sum = histogram('engine', e.name, 'result', 'count').sum
successful_count = counter('engine', e.name, 'search', 'count',
'successful')
result_count = int(result_count_sum /
float(successful_count)) if successful_count else 0
stats[e.name] = {
'time':
median,
'rate80':
rate80,
'rate95':
rate95,
'warn_timeout':
e.timeout > settings['outgoing']['request_timeout'],
'supports_selected_language':
_is_selected_language_supported(e, request.preferences),
'result_count':
result_count,
}
# end of stats
# reliabilities
reliabilities = {}
engine_errors = get_engine_errors(filtered_engines)
checker_results = checker_get_result()
checker_results = checker_results['engines'] \
if checker_results['status'] == 'ok' and 'engines' in checker_results else {}
for _, e in filtered_engines.items():
checker_result = checker_results.get(e.name, {})
checker_success = checker_result.get('success', True)
errors = engine_errors.get(e.name) or []
if counter('engine', e.name, 'search', 'count', 'sent') == 0:
# no request
reliablity = None
elif checker_success and not errors:
reliablity = 100
elif 'simple' in checker_result.get('errors', {}):
# the basic (simple) test doesn't work: the engine is broken accoding to the checker
# even if there is no exception
reliablity = 0
else:
reliablity = 100 - sum([
error['percentage']
for error in errors if not error.get('secondary')
])
reliabilities[e.name] = {
'reliablity': reliablity,
'errors': [],
'checker': checker_results.get(e.name, {}).get('errors',
{}).keys(),
}
# keep the order of the list checker_results[e.name]['errors'] and deduplicate.
# the first element has the highest percentage rate.
reliabilities_errors = []
for error in errors:
error_user_text = None
if error.get('secondary') or 'exception_classname' not in error:
continue
error_user_text = exception_classname_to_text.get(
error.get('exception_classname'))
if not error:
error_user_text = exception_classname_to_text[None]
if error_user_text not in reliabilities_errors:
reliabilities_errors.append(error_user_text)
reliabilities[e.name]['errors'] = reliabilities_errors
# supports
supports = {}
for _, e in filtered_engines.items():
supports_selected_language = _is_selected_language_supported(
e, request.preferences)
safesearch = e.safesearch
time_range_support = e.time_range_support
for checker_test_name in checker_results.get(e.name,
{}).get('errors', {}):
if supports_selected_language and checker_test_name.startswith(
'lang_'):
supports_selected_language = '?'
elif safesearch and checker_test_name == 'safesearch':
safesearch = '?'
elif time_range_support and checker_test_name == 'time_range':
time_range_support = '?'
supports[e.name] = {
'supports_selected_language': supports_selected_language,
'safesearch': safesearch,
'time_range_support': time_range_support,
}
return render(
'preferences.html',
selected_categories=get_selected_categories(request.preferences,
request.form),
locales=settings['locales'],
current_locale=request.preferences.get_value("locale"),
image_proxy=image_proxy,
engines_by_category=engines_by_category,
stats=stats,
max_rate95=max_rate95,
reliabilities=reliabilities,
supports=supports,
answerers=[{
'info': a.self_info(),
'keywords': a.keywords
} for a in answerers],
disabled_engines=disabled_engines,
autocomplete_backends=autocomplete_backends,
shortcuts={y: x
for x, y in engine_shortcuts.items()},
themes=themes,
plugins=plugins,
doi_resolvers=settings['doi_resolvers'],
current_doi_resolver=get_doi_resolver(
request.args, request.preferences.get_value('doi_resolver')),
allowed_plugins=allowed_plugins,
theme=get_current_theme_name(),
preferences_url_params=request.preferences.get_as_url_params(),
locked_preferences=settings['preferences']['lock'],
preferences=True)