def register(self, *plugins):
for plugin in plugins:
for plugin_attr, plugin_attr_type in required_attrs:
if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin))
exit(3)
for plugin_attr, plugin_attr_type in optional_attrs:
if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
setattr(plugin, plugin_attr, plugin_attr_type())
plugin.id = plugin.name.replace(' ', '_')
self.plugins.append(plugin)
def register(self, *plugins):
for plugin in plugins:
for plugin_attr, plugin_attr_type in required_attrs:
if not hasattr(plugin, plugin_attr) or not isinstance(
getattr(plugin, plugin_attr), plugin_attr_type):
logger.critical(
'missing attribute "{0}", cannot load plugin: {1}'.
format(plugin_attr, plugin))
exit(3)
for plugin_attr, plugin_attr_type in optional_attrs:
if not hasattr(plugin, plugin_attr) or not isinstance(
getattr(plugin, plugin_attr), plugin_attr_type):
setattr(plugin, plugin_attr, plugin_attr_type())
plugin.id = plugin.name.replace(' ', '_')
self.plugins.append(plugin)
def load_external_plugins(plugin_names):
plugins = []
for name in plugin_names:
logger.debug('loading plugin: {0}'.format(name))
try:
pkg = import_module(name)
except Exception as e:
logger.critical('failed to load plugin module {0}: {1}'.format(name, e))
exit(3)
pkg.__base_path = dirname(abspath(pkg.__file__))
prepare_package_resources(pkg, name)
plugins.append(pkg)
logger.debug('plugin "{0}" loaded'.format(name))
return plugins
def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):
dep_path = join(base_path, resource_path)
file_name = basename(dep_path)
resource_path = join(target_dir, file_name)
if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path):
try:
copyfile(dep_path, resource_path)
# copy atime_ns and mtime_ns, so the weak ETags (generated by
# the HTTP server) do not change
dep_stat = stat(dep_path)
utime(resource_path, ns=(dep_stat.st_atime_ns, dep_stat.st_mtime_ns))
except:
logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name))
exit(3)
# returning with the web path of the resource
return join('plugins/external_plugins', plugin_dir, file_name)
def prepare_package_resources(pkg, name):
plugin_dir = 'plugin_' + name
target_dir = join(settings['ui']['static_path'],
'plugins/external_plugins', plugin_dir)
try:
makedirs(target_dir, exist_ok=True)
except:
logger.critical(
'failed to create resource directory {0} for plugin {1}'.format(
target_dir, name))
exit(3)
resources = []
if hasattr(pkg, 'js_dependencies'):
resources.extend(map(basename, pkg.js_dependencies))
pkg.js_dependencies = tuple([
sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
for x in pkg.js_dependencies
])
if hasattr(pkg, 'css_dependencies'):
resources.extend(map(basename, pkg.css_dependencies))
pkg.css_dependencies = tuple([
sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
for x in pkg.css_dependencies
])
for f in listdir(target_dir):
if basename(f) not in resources:
resource_path = join(target_dir, basename(f))
try:
remove(resource_path)
except:
logger.critical(
'failed to remove unused resource file {0} for plugin {1}'.
format(resource_path, name))
exit(3)
import json
import cStringIO
import os
import hashlib
import requests
from searx import logger
logger = logger.getChild('webapp')
try:
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
except:
logger.critical("cannot import dependency: pygments")
from sys import exit
exit(1)
from datetime import datetime, timedelta
from urllib import urlencode
from urlparse import urlparse, urljoin
from werkzeug.contrib.fixers import ProxyFix
from flask import (
Flask, request, render_template, url_for, Response, make_response,
redirect, send_from_directory
)
from flask.ext.babel import Babel, gettext, format_date
from searx import settings, searx_dir
from searx.engines import (
categories, engines, get_engines_stats, engine_shortcuts
import hmac
import json
import os
import sys
import requests
from searx import logger
logger = logger.getChild('webapp')
try:
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
except:
logger.critical("cannot import dependency: pygments")
from sys import exit
exit(1)
try:
from cgi import escape
except:
from html import escape
from six import next
from datetime import datetime, timedelta
from time import time
from werkzeug.middleware.proxy_fix import ProxyFix
from flask import (
Flask, request, render_template, url_for, Response, make_response,
redirect, send_from_directory
)
from babel.support import Translations
import json
import cStringIO
import os
import hashlib
import requests
from searx import logger
logger = logger.getChild('webapp')
try:
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
except:
logger.critical("cannot import dependency: pygments")
from sys import exit
exit(1)
from datetime import datetime, timedelta
from urllib import urlencode
from urlparse import urlparse, urljoin
from werkzeug.contrib.fixers import ProxyFix
from flask import (Flask, request, render_template, url_for, Response,
make_response, redirect, send_from_directory)
from flask.ext.babel import Babel, gettext, format_date
from searx import settings, searx_dir
from searx.engines import (categories, engines, get_engines_stats,
engine_shortcuts)
from searx.utils import (UnicodeWriter, highlight_content, html_to_text,
get_themes, get_static_files, get_result_templates,
import hmac
import json
import os
import sys
import requests
from searx import logger
logger = logger.getChild('webapp')
try:
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
except:
logger.critical("cannot import dependency: pygments")
from sys import exit
exit(1)
from cgi import escape
from datetime import datetime, timedelta
from werkzeug.contrib.fixers import ProxyFix
from flask import (
Flask, request, render_template, url_for, Response, make_response,
redirect, send_from_directory
)
from flask_babel import Babel, gettext, format_date, format_decimal
from flask.json import jsonify
from searx import settings, searx_dir, searx_debug
from searx.exceptions import SearxParameterException
from searx.engines import (
categories, engines, engine_shortcuts, get_engines_stats, initialize_engines
from searx.search.models import EngineRef, SearchQuery
from searx.search.processors import processors, initialize as initialize_processors
from searx.search.checker import initialize as initialize_checker
from searx.metrics import initialize as initialize_metrics, counter_inc, histogram_observe_time
logger = logger.getChild('search')
max_request_timeout = settings.get('outgoing', {}).get('max_request_timeout' or None)
if max_request_timeout is None:
logger.info('max_request_timeout={0}'.format(max_request_timeout))
else:
if isinstance(max_request_timeout, float):
logger.info('max_request_timeout={0} second(s)'.format(max_request_timeout))
else:
logger.critical('outgoing.max_request_timeout if defined has to be float')
import sys
sys.exit(1)
def initialize(settings_engines=None, enable_checker=False):
settings_engines = settings_engines or settings['engines']
initialize_processors(settings_engines)
initialize_metrics([engine['name'] for engine in settings_engines])
if enable_checker:
initialize_checker()
class Search:
"""Search information container"""
from searx import settings
from searx import logger
from searx.raise_for_httperror import raise_for_httperror
from collections import OrderedDict
import ast
logger = logger.getChild('poolrequests')
try:
import ssl
if ssl.OPENSSL_VERSION_INFO[0:3] < (1, 0, 2):
# https://github.com/certifi/python-certifi#1024-bit-root-certificates
logger.critical(
'You are using an old openssl version({0}), please upgrade above 1.0.2!'
.format(ssl.OPENSSL_VERSION))
sys.exit(1)
except ImportError:
ssl = None
if not getattr(ssl, "HAS_SNI", False):
try:
import OpenSSL # pylint: disable=unused-import
except ImportError:
logger.critical(
"ssl doesn't support SNI and the pyopenssl module is not installed.\n"
"Some HTTPS connections will fail")
sys.exit(1)
class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):
UnicodeWriter, highlight_content, html_to_text, get_themes,
get_static_files, get_result_templates, gen_useragent, dict_subset
)
from searx.version import VERSION_STRING
from searx.languages import language_codes
from searx.https_rewrite import https_url_rewrite
from searx.search import Search
from searx.query import Query
from searx.autocomplete import searx_bang, backends as autocomplete_backends
from searx import logger
try:
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
except:
logger.critical("cannot import dependency: pygments")
from sys import exit
exit(1)
logger = logger.getChild('webapp')
static_path, templates_path, themes =\
get_themes(settings['themes_path']
if settings.get('themes_path')
else searx_dir)
default_theme = settings['server'].get('default_theme', 'default')
static_files = get_static_files(searx_dir)
from searx.version import VERSION_STRING
from searx.languages import language_codes as languages
from searx.search import SearchWithPlugins, get_search_query_from_webapp
from searx.query import RawTextQuery
from searx.autocomplete import searx_bang, backends as autocomplete_backends
from searx.plugins import plugins
from searx.plugins.oa_doi_rewrite import get_doi_resolver
from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
from searx.answerers import answerers
# check if the pyopenssl package is installed.
# It is needed for SSL connection without trouble, see #298
try:
import OpenSSL.SSL # NOQA
except ImportError:
logger.critical("The pyopenssl package has to be installed.\n"
"Some HTTPS connections will fail")
# serve pages with HTTP/1.1
from werkzeug.serving import WSGIRequestHandler
WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
# about static
static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path'])
logger.debug('static directory is %s', static_path)
static_files = get_static_files(static_path)
# about templates
default_theme = settings['ui']['default_theme']
templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path'])
logger.debug('templates directory is %s', templates_path)
themes = get_themes(templates_path)