Пример #1
0
    def test_external_bang(self):
        query_text = '!!ddg the query'
        query = RawTextQuery(query_text, [])

        self.assertEqual(query.getFullQuery(), query_text)
        self.assertEqual(len(query.query_parts), 1)
        self.assertFalse(query.specific)
Пример #2
0
    def test_external_bang_not_found(self):
        query_text = '!!notfoundbang the query'
        query = RawTextQuery(query_text, [])

        self.assertEqual(query.getFullQuery(), query_text)
        self.assertEqual(query.external_bang, None)
        self.assertFalse(query.specific)
Пример #3
0
    def test_simple_query(self):
        query_text = 'the query'
        query = RawTextQuery(query_text, [])

        self.assertEqual(query.getFullQuery(), query_text)
        self.assertEqual(len(query.query_parts), 1)
        self.assertEqual(len(query.languages), 0)
        self.assertFalse(query.specific)
Пример #4
0
    def test_bang_autocomplete_empty(self):
        initialize()
        query = RawTextQuery('the query !', [])
        self.assertEqual(query.autocomplete_list,
                         ['!images', '!wikipedia', '!osm'])

        query = RawTextQuery('the query ?', ['osm'])
        self.assertEqual(query.autocomplete_list, ['?images', '?wikipedia'])
Пример #5
0
    def test_empty_colon_in_query(self):
        query_text = 'the : query'
        query = RawTextQuery(query_text, [])

        self.assertEqual(query.getFullQuery(), query_text)
        self.assertEqual(len(query.query_parts), 0)
        self.assertEqual(len(query.languages), 0)
        self.assertFalse(query.specific)
Пример #6
0
    def test_bang_autocomplete_empty(self):
        load_engines(settings['engines'])
        query = RawTextQuery('the query !', [])
        self.assertEqual(query.autocomplete_list,
                         ['!images', '!wikipedia', '!osm'])

        query = RawTextQuery('the query ?', ['osm'])
        self.assertEqual(query.autocomplete_list, ['?images', '?wikipedia'])
Пример #7
0
    def test_bang_autocomplete(self):
        initialize(TEST_ENGINES)
        query = RawTextQuery('the query !dum', [])
        self.assertEqual(query.autocomplete_list, ['!dummy_engine'])

        query = RawTextQuery('!dum the query', [])
        self.assertEqual(query.autocomplete_list, [])
        self.assertEqual(query.getQuery(), '!dum the query')
Пример #8
0
    def test_timeout_above1000(self):
        query_text = '<3500 the query'
        query = RawTextQuery(query_text, [])

        self.assertEqual(query.getFullQuery(), query_text)
        self.assertEqual(len(query.query_parts), 3)
        self.assertEqual(query.timeout_limit, 3.5)
        self.assertFalse(query.specific)
Пример #9
0
    def test_timeout_below100(self):
        query_text = '<3 the query'
        query = RawTextQuery(query_text, [])
        query.parse_query()

        self.assertEquals(query.getFullQuery(), query_text)
        self.assertEquals(len(query.query_parts), 3)
        self.assertEquals(query.timeout_limit, 3)
        self.assertFalse(query.specific)
Пример #10
0
    def test_multiple_spaces_query(self):
        query_text = '\tthe   query'
        query = RawTextQuery(query_text, [])

        self.assertEqual(query.getFullQuery(), 'the query')
        self.assertEqual(len(query.query_parts), 0)
        self.assertEqual(len(query.user_query_parts), 2)
        self.assertEqual(len(query.languages), 0)
        self.assertFalse(query.specific)
Пример #11
0
    def test_simple_query(self):
        query_text = 'the query'
        query = RawTextQuery(query_text, [])
        query.parse_query()

        self.assertEquals(query.getFullQuery(), query_text)
        self.assertEquals(len(query.query_parts), 1)
        self.assertEquals(len(query.languages), 0)
        self.assertFalse(query.specific)
Пример #12
0
    def test_invalid_language_code(self):
        language = 'not_a_language'
        query_text = 'the query'
        full_query = ':' + language + ' ' + query_text
        query = RawTextQuery(full_query, [])

        self.assertEqual(query.getFullQuery(), full_query)
        self.assertEqual(len(query.query_parts), 1)
        self.assertEqual(len(query.languages), 0)
        self.assertFalse(query.specific)
Пример #13
0
    def test_unlisted_language_code(self):
        language = 'all'
        query_text = 'the query'
        full_query = ':' + language + ' ' + query_text
        query = RawTextQuery(full_query, [])

        self.assertEqual(query.getFullQuery(), full_query)
        self.assertEqual(len(query.query_parts), 3)
        self.assertIn('all', query.languages)
        self.assertFalse(query.specific)
Пример #14
0
    def test_language_name(self):
        language = 'english'
        query_text = 'the query'
        full_query = ':' + language + ' ' + query_text
        query = RawTextQuery(full_query, [])

        self.assertEqual(query.getFullQuery(), full_query)
        self.assertEqual(len(query.query_parts), 1)
        self.assertIn('en', query.languages)
        self.assertFalse(query.specific)
Пример #15
0
    def test_timeout_invalid(self):
        # invalid number: it is not bang but it is part of the query
        query_text = '<xxx the query'
        query = RawTextQuery(query_text, [])

        self.assertEqual(query.getFullQuery(), query_text)
        self.assertEqual(len(query.query_parts), 1)
        self.assertEqual(query.query_parts[0], query_text)
        self.assertEqual(query.timeout_limit, None)
        self.assertFalse(query.specific)
Пример #16
0
    def test_invalid_language_code(self):
        language = 'not_a_language'
        query_text = 'the query'
        full_query = ':' + language + ' ' + query_text
        query = RawTextQuery(full_query, [])
        query.parse_query()

        self.assertEquals(query.getFullQuery(), full_query)
        self.assertEquals(len(query.query_parts), 1)
        self.assertEquals(len(query.languages), 0)
        self.assertFalse(query.specific)
Пример #17
0
    def test_bang_autocomplete_empty(self):
        with patch.object(searx.engines, 'initialize_engines',
                          searx.engines.load_engines):
            initialize()
            query = RawTextQuery('the query !', [])
            self.assertEqual(query.autocomplete_list,
                             ['!images', '!wikipedia', '!osm'])

            query = RawTextQuery('the query ?', ['osm'])
            self.assertEqual(query.autocomplete_list,
                             ['?images', '?wikipedia'])
Пример #18
0
    def test_unlisted_language_code(self):
        language = 'all'
        query_text = 'the query'
        full_query = ':' + language + ' ' + query_text
        query = RawTextQuery(full_query, [])
        query.parse_query()

        self.assertEquals(query.getFullQuery(), full_query)
        self.assertEquals(len(query.query_parts), 3)
        self.assertIn('all', query.languages)
        self.assertFalse(query.specific)
Пример #19
0
    def test_timeout_autocomplete(self):
        # invalid number: it is not bang but it is part of the query
        query_text = 'the query <'
        query = RawTextQuery(query_text, [])

        self.assertEqual(query.getFullQuery(), query_text)
        self.assertEqual(len(query.query_parts), 0)
        self.assertEqual(query.getQuery(), query_text)
        self.assertEqual(query.timeout_limit, None)
        self.assertFalse(query.specific)
        self.assertEqual(query.autocomplete_list, ['<3', '<850'])
Пример #20
0
    def test_language_code(self):
        language = 'es-ES'
        query_text = 'the query'
        full_query = ':' + language + ' ' + query_text
        query = RawTextQuery(full_query, [])
        query.parse_query()

        self.assertEquals(query.getFullQuery(), full_query)
        self.assertEquals(len(query.query_parts), 3)
        self.assertEquals(len(query.languages), 1)
        self.assertIn(language, query.languages)
        self.assertFalse(query.specific)
Пример #21
0
    def test_external_bang_autocomplete_empty(self):
        query_text = 'the query !!'
        query = RawTextQuery(query_text, [])

        self.assertEqual(query.getFullQuery(), 'the query !!')
        self.assertEqual(len(query.query_parts), 0)
        self.assertFalse(query.specific)
        self.assertGreater(len(query.autocomplete_list), 2)

        a = query.autocomplete_list[0]
        self.assertEqual(query.get_autocomplete_full_query(a),
                         'the query ' + a)
Пример #22
0
def autocompleter():
    """Return autocompleter results"""

    # run autocompleter
    results = []

    # set blocked engines
    disabled_engines = request.preferences.engines.get_disabled()

    # parse query
    raw_text_query = RawTextQuery(request.form.get('q', ''), disabled_engines)
    sug_prefix = raw_text_query.getQuery()

    # normal autocompletion results only appear if no inner results returned
    # and there is a query part
    if len(raw_text_query.autocomplete_list) == 0 and len(sug_prefix) > 0:

        # get language from cookie
        language = request.preferences.get_value('language')
        if not language or language == 'all':
            language = 'en'
        else:
            language = language.split('-')[0]

        # run autocompletion
        raw_results = search_autocomplete(
            request.preferences.get_value('autocomplete'), sug_prefix,
            language)
        for result in raw_results:
            # attention: this loop will change raw_text_query object and this is
            # the reason why the sug_prefix was stored before (see above)
            results.append(raw_text_query.changeQuery(result).getFullQuery())

    if len(raw_text_query.autocomplete_list) > 0:
        for autocomplete_text in raw_text_query.autocomplete_list:
            results.append(
                raw_text_query.get_autocomplete_full_query(autocomplete_text))

    for answers in ask(raw_text_query):
        for answer in answers:
            results.append(str(answer['answer']))

    if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
        # the suggestion request comes from the searx search form
        suggestions = json.dumps(results)
        mimetype = 'application/json'
    else:
        # the suggestion request comes from browser's URL bar
        suggestions = json.dumps([sug_prefix, results])
        mimetype = 'application/x-suggestions+json'

    return Response(suggestions, mimetype=mimetype)
Пример #23
0
    def test_bang(self):
        initialize(TEST_ENGINES)

        for bang in TestBang.SPECIFIC_BANGS + TestBang.NOT_SPECIFIC_BANGS:
            with self.subTest(msg="Check bang", bang=bang):
                query_text = TestBang.THE_QUERY + ' ' + bang
                query = RawTextQuery(query_text, [])

                self.assertEqual(query.getFullQuery(),
                                 bang + ' ' + TestBang.THE_QUERY)
                self.assertEqual(query.query_parts, [bang])
                self.assertEqual(query.user_query_parts,
                                 TestBang.THE_QUERY.split(' '))
Пример #24
0
    def test_autocomplete(self):
        query = RawTextQuery(':englis', [])
        self.assertEqual(query.autocomplete_list, [":english"])

        query = RawTextQuery(':deutschla', [])
        self.assertEqual(query.autocomplete_list, [":deutschland"])

        query = RawTextQuery(':new_zea', [])
        self.assertEqual(query.autocomplete_list, [":new_zealand"])

        query = RawTextQuery(':hu-H', [])
        self.assertEqual(query.autocomplete_list, [":hu-hu"])

        query = RawTextQuery(':v', [])
        self.assertEqual(query.autocomplete_list, [":vi", ":tiếng việt"])
def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])\
        -> Tuple[SearchQuery, RawTextQuery, List[EngineRef], List[EngineRef]]:
    # no text for the query ?
    if not form.get('q'):
        raise SearxParameterException('q', '')

    # set blocked engines
    disabled_engines = preferences.engines.get_disabled()

    # parse query, if tags are set, which change
    # the serch engine or search-language
    raw_text_query = RawTextQuery(form['q'], disabled_engines)

    # set query
    query = raw_text_query.getQuery()
    query_pageno = parse_pageno(form)
    query_lang = parse_lang(preferences, form, raw_text_query)
    query_safesearch = parse_safesearch(preferences, form)
    query_time_range = parse_time_range(form)
    query_timeout = parse_timeout(form, raw_text_query)
    external_bang = raw_text_query.external_bang

    if not is_locked(
            'categories'
    ) and raw_text_query.enginerefs and raw_text_query.specific:
        # if engines are calculated from query,
        # set categories by using that informations
        query_engineref_list = raw_text_query.enginerefs
    else:
        # otherwise, using defined categories to
        # calculate which engines should be used
        query_engineref_list = parse_generic(preferences, form,
                                             disabled_engines)

    query_engineref_list = deduplicate_engineref_list(query_engineref_list)
    query_engineref_list, query_engineref_list_unknown, query_engineref_list_notoken =\
        validate_engineref_list(query_engineref_list, preferences)

    return (SearchQuery(query,
                        query_engineref_list,
                        query_lang,
                        query_safesearch,
                        query_pageno,
                        query_time_range,
                        query_timeout,
                        external_bang=external_bang), raw_text_query,
            query_engineref_list_unknown, query_engineref_list_notoken)
Пример #26
0
def autocompleter():
    """Return autocompleter results"""

    # set blocked engines
    disabled_engines = request.preferences.engines.get_disabled()

    # parse query
    if PY3:
        raw_text_query = RawTextQuery(request.form.get('q', b''), disabled_engines)
    else:
        raw_text_query = RawTextQuery(request.form.get('q', u'').encode('utf-8'), disabled_engines)
    raw_text_query.parse_query()

    # check if search query is set
    if not raw_text_query.getSearchQuery():
        return '', 400

    # run autocompleter
    completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))

    # parse searx specific autocompleter results like !bang
    raw_results = searx_bang(raw_text_query)

    # normal autocompletion results only appear if max 3 inner results returned
    if len(raw_results) <= 3 and completer:
        # get language from cookie
        language = request.preferences.get_value('language')
        if not language or language == 'all':
            language = 'en'
        else:
            language = language.split('-')[0]
        # run autocompletion
        raw_results.extend(completer(raw_text_query.getSearchQuery(), language))

    # parse results (write :language and !engine back to result string)
    results = []
    for result in raw_results:
        raw_text_query.changeSearchQuery(result)

        # add parsed result
        results.append(raw_text_query.getFullQuery())

    # return autocompleter results
    if request.form.get('format') == 'x-suggestions':
        return Response(json.dumps([raw_text_query.query, results]),
                        mimetype='application/json')

    return Response(json.dumps(results),
                    mimetype='application/json')
Пример #27
0
def autocompleter():
    """Return autocompleter results"""

    # set blocked engines
    disabled_engines = request.preferences.engines.get_disabled()

    # parse query
    raw_text_query = RawTextQuery(
        request.form.get('q', '').encode('utf-8'), disabled_engines)
    raw_text_query.parse_query()

    # check if search query is set
    if not raw_text_query.getSearchQuery():
        return '', 400

    # run autocompleter
    completer = autocomplete_backends.get(
        request.preferences.get_value('autocomplete'))

    # parse searx specific autocompleter results like !bang
    raw_results = searx_bang(raw_text_query)

    # normal autocompletion results only appear if max 3 inner results returned
    if len(raw_results) <= 3 and completer:
        # get language from cookie
        language = request.preferences.get_value('language')
        if not language or language == 'all':
            language = 'en'
        else:
            language = language.split('_')[0]
        # run autocompletion
        raw_results.extend(completer(raw_text_query.getSearchQuery(),
                                     language))

    # parse results (write :language and !engine back to result string)
    results = []
    for result in raw_results:
        raw_text_query.changeSearchQuery(result)

        # add parsed result
        results.append(raw_text_query.getFullQuery())

    # return autocompleter results
    if request.form.get('format') == 'x-suggestions':
        return Response(json.dumps([raw_text_query.query, results]),
                        mimetype='application/json')

    return Response(json.dumps(results), mimetype='application/json')
Пример #28
0
def autocompleter():
    """Return autocompleter results"""

    # set blocked engines
    disabled_engines = request.preferences.engines.get_disabled()

    # parse query
    raw_text_query = RawTextQuery(request.form.get("q", ""), disabled_engines)

    # check if search query is set
    if not raw_text_query.getQuery():
        return "", 400

    # run autocompleter
    completer = autocomplete_backends.get(
        request.preferences.get_value("autocomplete"))

    # parse searx specific autocompleter results like !bang
    raw_results = searx_bang(raw_text_query)

    # normal autocompletion results only appear if no inner results returned
    # and there is a query part besides the engine and language bangs
    if (len(raw_results) == 0 and completer
            and (len(raw_text_query.query_parts) > 1 or
                 (len(raw_text_query.languages) == 0
                  and not raw_text_query.specific))):
        # get language from cookie
        language = request.preferences.get_value("language")
        if not language or language == "all":
            language = "en"
        else:
            language = language.split("-")[0]
        # run autocompletion
        raw_results.extend(completer(raw_text_query.getQuery(), language))

    # parse results (write :language and !engine back to result string)
    results = []
    for result in raw_results:
        raw_text_query.changeQuery(result)

        # add parsed result
        results.append(raw_text_query.getFullQuery())

    # return autocompleter results
    if request.headers.get("X-Requested-With") == "XMLHttpRequest":
        return Response(json.dumps(results), mimetype="application/json")

    return Response(
        json.dumps([raw_text_query.query, results]),
        mimetype="application/x-suggestions+json",
    )
Пример #29
0
def autocompleter():
    """Return autocompleter results"""

    # set blocked engines
    disabled_engines = request.preferences.engines.get_disabled()

    # parse query
    raw_text_query = RawTextQuery(request.form.get("q", "").encode("utf-8"), disabled_engines)
    raw_text_query.parse_query()

    # check if search query is set
    if not raw_text_query.getSearchQuery():
        return "", 400

    # run autocompleter
    completer = autocomplete_backends.get(request.preferences.get_value("autocomplete"))

    # parse searx specific autocompleter results like !bang
    raw_results = searx_bang(raw_text_query)

    # normal autocompletion results only appear if max 3 inner results returned
    if len(raw_results) <= 3 and completer:
        # get language from cookie
        language = request.preferences.get_value("language")
        if not language or language == "all":
            language = "en"
        else:
            language = language.split("_")[0]
        # run autocompletion
        raw_results.extend(completer(raw_text_query.getSearchQuery(), language))

    # parse results (write :language and !engine back to result string)
    results = []
    for result in raw_results:
        raw_text_query.changeSearchQuery(result)

        # add parsed result
        results.append(raw_text_query.getFullQuery())

    # return autocompleter results
    if request.form.get("format") == "x-suggestions":
        return Response(json.dumps([raw_text_query.query, results]), mimetype="application/json")

    return Response(json.dumps(results), mimetype="application/json")
Пример #30
0
def autocompleter():
    """Return autocompleter results"""

    # run autocompleter
    results = []

    # set blocked engines
    disabled_engines = request.preferences.engines.get_disabled()

    # parse query
    raw_text_query = RawTextQuery(request.form.get('q', ''), disabled_engines)

    # normal autocompletion results only appear if no inner results returned
    # and there is a query part
    if len(raw_text_query.autocomplete_list) == 0 and len(
            raw_text_query.getQuery()) > 0:
        # get language from cookie
        language = request.preferences.get_value('language')
        if not language or language == 'all':
            language = 'en'
        else:
            language = language.split('-')[0]
        # run autocompletion
        raw_results = search_autocomplete(
            request.preferences.get_value('autocomplete'),
            raw_text_query.getQuery(), language)
        for result in raw_results:
            results.append(raw_text_query.changeQuery(result).getFullQuery())

    if len(raw_text_query.autocomplete_list) > 0:
        for autocomplete_text in raw_text_query.autocomplete_list:
            results.append(
                raw_text_query.get_autocomplete_full_query(autocomplete_text))

    for answers in ask(raw_text_query):
        for answer in answers:
            results.append(str(answer['answer']))

    # return autocompleter results
    if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
        return Response(json.dumps(results), mimetype='application/json')

    return Response(json.dumps([raw_text_query.query, results]),
                    mimetype='application/x-suggestions+json')
Пример #31
0
def get_search_query_from_webapp(preferences, form):
    # no text for the query ?
    if not form.get('q'):
        raise SearxParameterException('q', '')

    # set blocked engines
    disabled_engines = preferences.engines.get_disabled()

    # parse query, if tags are set, which change
    # the serch engine or search-language
    raw_text_query = RawTextQuery(form['q'], disabled_engines)
    raw_text_query.parse_query()

    # set query
    query = raw_text_query.getSearchQuery()

    # get and check page number
    pageno_param = form.get('pageno', '1')
    if not pageno_param.isdigit() or int(pageno_param) < 1:
        raise SearxParameterException('pageno', pageno_param)
    query_pageno = int(pageno_param)

    # get language
    # set specific language if set on request, query or preferences
    # TODO support search with multible languages
    if len(raw_text_query.languages):
        query_lang = raw_text_query.languages[-1]
    elif 'language' in form:
        query_lang = form.get('language')
    else:
        query_lang = preferences.get_value('language')

    # provides backwards compatibility for requests using old language default
    if query_lang == 'all':
        query_lang = settings['search']['language']

    # check language
    if not VALID_LANGUAGE_CODE.match(query_lang):
        raise SearxParameterException('language', query_lang)

    # get safesearch
    if 'safesearch' in form:
        query_safesearch = form.get('safesearch')
        # first check safesearch
        if not query_safesearch.isdigit():
            raise SearxParameterException('safesearch', query_safesearch)
        query_safesearch = int(query_safesearch)
    else:
        query_safesearch = preferences.get_value('safesearch')

    # safesearch : second check
    if query_safesearch < 0 or query_safesearch > 2:
        raise SearxParameterException('safesearch', query_safesearch)

    # get time_range
    query_time_range = form.get('time_range')

    # check time_range
    if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
        raise SearxParameterException('time_range', query_time_range)

    # query_engines
    query_engines = raw_text_query.engines

    # query_categories
    query_categories = []

    # if engines are calculated from query,
    # set categories by using that informations
    if query_engines and raw_text_query.specific:
        query_categories = list(set(engine['category']
                                    for engine in query_engines))

    # otherwise, using defined categories to
    # calculate which engines should be used
    else:
        # set categories/engines
        load_default_categories = True
        for pd_name, pd in form.items():
            if pd_name == 'categories':
                query_categories.extend(categ for categ in map(unicode.strip, pd.split(',')) if categ in categories)
            elif pd_name == 'engines':
                pd_engines = [{'category': engines[engine].categories[0],
                               'name': engine}
                              for engine in map(unicode.strip, pd.split(',')) if engine in engines]
                if pd_engines:
                    query_engines.extend(pd_engines)
                    load_default_categories = False
            elif pd_name.startswith('category_'):
                category = pd_name[9:]

                # if category is not found in list, skip
                if category not in categories:
                    continue

                if pd != 'off':
                    # add category to list
                    query_categories.append(category)
                elif category in query_categories:
                    # remove category from list if property is set to 'off'
                    query_categories.remove(category)

        if not load_default_categories:
            if not query_categories:
                query_categories = list(set(engine['category']
                                            for engine in query_engines))
        else:
            # if no category is specified for this search,
            # using user-defined default-configuration which
            # (is stored in cookie)
            if not query_categories:
                cookie_categories = preferences.get_value('categories')
                for ccateg in cookie_categories:
                    if ccateg in categories:
                        query_categories.append(ccateg)

            # if still no category is specified, using general
            # as default-category
            if not query_categories:
                query_categories = ['general']

            # using all engines for that search, which are
            # declared under the specific categories
            for categ in query_categories:
                query_engines.extend({'category': categ,
                                      'name': engine.name}
                                     for engine in categories[categ]
                                     if (engine.name, categ) not in disabled_engines)

    return SearchQuery(query, query_engines, query_categories,
                       query_lang, query_safesearch, query_pageno, query_time_range)
Пример #32
0
 def test_repr_method(self):
     query_text = '<8 the query'
     query = RawTextQuery(query_text, [])
     r = repr(query)
     self.assertTrue(r.startswith(f"<RawTextQuery query='{query_text}' "))
Пример #33
0
def autocompleter():
    """Return autocompleter results"""

    # set blocked engines
    disabled_engines = request.preferences.engines.get_disabled()

    # parse query
    if PY3:
        raw_text_query = RawTextQuery(request.form.get('q', b''),
                                      disabled_engines)
    else:
        raw_text_query = RawTextQuery(
            request.form.get('q', u'').encode('utf-8'), disabled_engines)
    raw_text_query.parse_query()

    # check if search query is set
    if not raw_text_query.getSearchQuery():
        return '', 400

    # run autocompleter
    completer = autocomplete_backends.get(
        request.preferences.get_value('autocomplete'))

    # parse searx specific autocompleter results like !bang
    raw_results = searx_bang(raw_text_query)

    # normal autocompletion results only appear if no inner results returned
    # and there is a query part besides the engine and language bangs
    if len(raw_results) == 0 and completer and (
            len(raw_text_query.query_parts) > 1 or
        (len(raw_text_query.languages) == 0 and not raw_text_query.specific)):
        # get language from cookie
        language = request.preferences.get_value('language')
        if not language or language == 'all':
            language = 'en'
        else:
            language = language.split('-')[0]
        # run autocompletion
        raw_results.extend(completer(raw_text_query.getSearchQuery(),
                                     language))

    # parse results (write :language and !engine back to result string)
    results = []
    for result in raw_results:
        raw_text_query.changeSearchQuery(result)

        # add parsed result
        results.append(raw_text_query.getFullQuery())

    # return autocompleter results
    if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
        return Response(json.dumps(results), mimetype='application/json')

    return Response(json.dumps([raw_text_query.query, results]),
                    mimetype='application/x-suggestions+json')
Пример #34
0
def get_search_query_from_webapp(preferences, form):
    query = None
    query_engines = []
    query_categories = []
    query_paging = False
    query_pageno = 1
    query_lang = 'all'
    query_time_range = None

    # set blocked engines
    disabled_engines = preferences.engines.get_disabled()

    # set specific language if set
    query_lang = preferences.get_value('language')

    # safesearch
    query_safesearch = preferences.get_value('safesearch')

    # TODO better exceptions
    if not form.get('q'):
        raise Exception('noquery')

    # set pagenumber
    pageno_param = form.get('pageno', '1')
    if not pageno_param.isdigit() or int(pageno_param) < 1:
        pageno_param = 1

    query_pageno = int(pageno_param)

    # parse query, if tags are set, which change
    # the serch engine or search-language
    raw_text_query = RawTextQuery(form['q'], disabled_engines)
    raw_text_query.parse_query()

    # set query
    query = raw_text_query.getSearchQuery()

    # get last selected language in query, if possible
    # TODO support search with multible languages
    if len(raw_text_query.languages):
        query_lang = raw_text_query.languages[-1]

    query_time_range = form.get('time_range')

    query_engines = raw_text_query.engines

    # if engines are calculated from query,
    # set categories by using that informations
    if query_engines and raw_text_query.specific:
        query_categories = list(set(engine['category']
                                    for engine in query_engines))

    # otherwise, using defined categories to
    # calculate which engines should be used
    else:
        # set categories/engines
        load_default_categories = True
        for pd_name, pd in form.items():
            if pd_name == 'categories':
                query_categories.extend(categ for categ in map(unicode.strip, pd.split(',')) if categ in categories)
            elif pd_name == 'engines':
                pd_engines = [{'category': engines[engine].categories[0],
                               'name': engine}
                              for engine in map(unicode.strip, pd.split(',')) if engine in engines]
                if pd_engines:
                    query_engines.extend(pd_engines)
                    load_default_categories = False
            elif pd_name.startswith('category_'):
                category = pd_name[9:]

                # if category is not found in list, skip
                if category not in categories:
                    continue

                if pd != 'off':
                    # add category to list
                    query_categories.append(category)
                elif category in query_categories:
                    # remove category from list if property is set to 'off'
                    query_categories.remove(category)

        if not load_default_categories:
            if not query_categories:
                query_categories = list(set(engine['category']
                                            for engine in engines))
        else:
            # if no category is specified for this search,
            # using user-defined default-configuration which
            # (is stored in cookie)
            if not query_categories:
                cookie_categories = preferences.get_value('categories')
                for ccateg in cookie_categories:
                    if ccateg in categories:
                        query_categories.append(ccateg)

            # if still no category is specified, using general
            # as default-category
            if not query_categories:
                query_categories = ['general']

            # using all engines for that search, which are
            # declared under the specific categories
            for categ in query_categories:
                query_engines.extend({'category': categ,
                                      'name': engine.name}
                                     for engine in categories[categ]
                                     if (engine.name, categ) not in disabled_engines)

    return SearchQuery(query, query_engines, query_categories,
                       query_lang, query_safesearch, query_pageno, query_time_range)
Пример #35
0
 def test_change_query(self):
     query_text = '<8 the query'
     query = RawTextQuery(query_text, [])
     another_query = query.changeQuery('another text')
     self.assertEqual(query, another_query)
     self.assertEqual(query.getFullQuery(), '<8 another text')
Пример #36
0
def get_search_query_from_webapp(preferences, form):
    # no text for the query ?
    if not form.get('q'):
        raise SearxParameterException('q', '')

    # set blocked engines
    disabled_engines = preferences.engines.get_disabled()

    # parse query, if tags are set, which change
    # the serch engine or search-language
    raw_text_query = RawTextQuery(form['q'], disabled_engines)
    raw_text_query.parse_query()

    # set query
    query = raw_text_query.getSearchQuery()

    # get and check page number
    pageno_param = form.get('pageno', '1')
    if not pageno_param.isdigit() or int(pageno_param) < 1:
        raise SearxParameterException('pageno', pageno_param)
    query_pageno = int(pageno_param)

    # get language
    # set specific language if set on request, query or preferences
    # TODO support search with multible languages
    if len(raw_text_query.languages):
        query_lang = raw_text_query.languages[-1]
    elif 'language' in form:
        query_lang = form.get('language')
    else:
        query_lang = preferences.get_value('language')

    # check language
    if not VALID_LANGUAGE_CODE.match(query_lang):
        raise SearxParameterException('language', query_lang)

    # get safesearch
    if 'safesearch' in form:
        query_safesearch = form.get('safesearch')
        # first check safesearch
        if not query_safesearch.isdigit():
            raise SearxParameterException('safesearch', query_safesearch)
        query_safesearch = int(query_safesearch)
    else:
        query_safesearch = preferences.get_value('safesearch')

    # safesearch : second check
    if query_safesearch < 0 or query_safesearch > 2:
        raise SearxParameterException('safesearch', query_safesearch)

    # get time_range
    query_time_range = form.get('time_range')

    # check time_range
    if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
        raise SearxParameterException('time_range', query_time_range)

    # query_engines
    query_engines = raw_text_query.engines

    # query_categories
    query_categories = []

    # if engines are calculated from query,
    # set categories by using that informations
    if query_engines and raw_text_query.specific:
        query_categories = list(set(engine['category']
                                    for engine in query_engines))

    # otherwise, using defined categories to
    # calculate which engines should be used
    else:
        # set categories/engines
        load_default_categories = True
        for pd_name, pd in form.items():
            if pd_name == 'categories':
                query_categories.extend(categ for categ in map(unicode.strip, pd.split(',')) if categ in categories)
            elif pd_name == 'engines':
                pd_engines = [{'category': engines[engine].categories[0],
                               'name': engine}
                              for engine in map(unicode.strip, pd.split(',')) if engine in engines]
                if pd_engines:
                    query_engines.extend(pd_engines)
                    load_default_categories = False
            elif pd_name.startswith('category_'):
                category = pd_name[9:]

                # if category is not found in list, skip
                if category not in categories:
                    continue

                if pd != 'off':
                    # add category to list
                    query_categories.append(category)
                elif category in query_categories:
                    # remove category from list if property is set to 'off'
                    query_categories.remove(category)

        if not load_default_categories:
            if not query_categories:
                query_categories = list(set(engine['category']
                                            for engine in query_engines))
        else:
            # if no category is specified for this search,
            # using user-defined default-configuration which
            # (is stored in cookie)
            if not query_categories:
                cookie_categories = preferences.get_value('categories')
                for ccateg in cookie_categories:
                    if ccateg in categories:
                        query_categories.append(ccateg)

            # if still no category is specified, using general
            # as default-category
            if not query_categories:
                query_categories = ['general']

            # using all engines for that search, which are
            # declared under the specific categories
            for categ in query_categories:
                query_engines.extend({'category': categ,
                                      'name': engine.name}
                                     for engine in categories[categ]
                                     if (engine.name, categ) not in disabled_engines)

    return SearchQuery(query, query_engines, query_categories,
                       query_lang, query_safesearch, query_pageno, query_time_range)
Пример #37
0
def get_search_query_from_webapp(preferences, form):
    query = None
    query_engines = []
    query_categories = []
    query_paging = False
    query_pageno = 1
    query_lang = 'all'
    query_time_range = None

    # set blocked engines
    disabled_engines = preferences.engines.get_disabled()

    # set specific language if set
    query_lang = preferences.get_value('language')

    # safesearch
    query_safesearch = preferences.get_value('safesearch')

    # TODO better exceptions
    if not form.get('q'):
        raise Exception('noquery')

    # set pagenumber
    pageno_param = form.get('pageno', '1')
    if not pageno_param.isdigit() or int(pageno_param) < 1:
        pageno_param = 1

    query_pageno = int(pageno_param)

    # parse query, if tags are set, which change
    # the serch engine or search-language
    raw_text_query = RawTextQuery(form['q'], disabled_engines)
    raw_text_query.parse_query()

    # set query
    query = raw_text_query.getSearchQuery()

    # get last selected language in query, if possible
    # TODO support search with multible languages
    if len(raw_text_query.languages):
        query_lang = raw_text_query.languages[-1]

    query_time_range = form.get('time_range')

    query_engines = raw_text_query.engines

    # if engines are calculated from query,
    # set categories by using that informations
    if query_engines and raw_text_query.specific:
        query_categories = list(
            set(engine['category'] for engine in query_engines))

    # otherwise, using defined categories to
    # calculate which engines should be used
    else:
        # set categories/engines
        load_default_categories = True
        for pd_name, pd in form.items():
            if pd_name == 'categories':
                query_categories.extend(
                    categ for categ in map(unicode.strip, pd.split(','))
                    if categ in categories)
            elif pd_name == 'engines':
                pd_engines = [{
                    'category': engines[engine].categories[0],
                    'name': engine
                } for engine in map(unicode.strip, pd.split(','))
                              if engine in engines]
                if pd_engines:
                    query_engines.extend(pd_engines)
                    load_default_categories = False
            elif pd_name.startswith('category_'):
                category = pd_name[9:]

                # if category is not found in list, skip
                if category not in categories:
                    continue

                if pd != 'off':
                    # add category to list
                    query_categories.append(category)
                elif category in query_categories:
                    # remove category from list if property is set to 'off'
                    query_categories.remove(category)

        if not load_default_categories:
            if not query_categories:
                query_categories = list(
                    set(engine['category'] for engine in engines))
        else:
            # if no category is specified for this search,
            # using user-defined default-configuration which
            # (is stored in cookie)
            if not query_categories:
                cookie_categories = preferences.get_value('categories')
                for ccateg in cookie_categories:
                    if ccateg in categories:
                        query_categories.append(ccateg)

            # if still no category is specified, using general
            # as default-category
            if not query_categories:
                query_categories = ['general']

            # using all engines for that search, which are
            # declared under the specific categories
            for categ in query_categories:
                query_engines.extend({
                    'category': categ,
                    'name': engine.name
                } for engine in categories[categ]
                                     if (engine.name,
                                         categ) not in disabled_engines)

    return SearchQuery(query, query_engines, query_categories, query_lang,
                       query_safesearch, query_pageno, query_time_range)