def request(query, params): offset = (params['pageno'] - 1) * 10 if categories[0] and categories[0] in category_to_keyword: params['url'] = url.format(keyword=category_to_keyword[categories[0]], query=urlencode({'q': query}), offset=offset) else: params['url'] = url.format(keyword='web', query=urlencode({'q': query}), offset=offset) # add language tag if params['language'] == 'no' or params['language'].startswith('no-'): params['language'] = params['language'].replace('no', 'nb', 1) if params['language'].find('-') < 0: # tries to get a country code from language for lang in supported_languages: lc = lang.split('-') if params['language'] == lc[0]: params['language'] = lang break params['url'] += '&locale=' + params['language'].replace('-', '_').lower() return params
def request(query, params): # obtain token if last update was more than an hour if time() - (token['last_updated'] or 0) > 3600: obtain_token() params['url'] = search_url.format(query=urlencode({'input': query}), token=token['value']) params['headers']['Referer'] = referer_url.format(query=urlencode({'i': query})) return params
def _get_url(query, offset, language, time_range): if time_range in time_range_dict: return base_url + search_url_with_time.format(offset=offset, query=urlencode({'p': query}), lang=language, age=time_range_dict[time_range][0], btf=time_range_dict[time_range][1]) return base_url + search_url.format(offset=offset, query=urlencode({'p': query}), lang=language)
def _get_url(query, language, offset, time_range): if time_range in time_range_dict: search_path = search_string_with_time.format( query=urlencode({'q': query, 'setmkt': language}), offset=offset, interval=time_range_dict[time_range]) else: search_path = search_string.format( query=urlencode({'q': query, 'setmkt': language}), offset=offset) return base_url + search_path
def request(query, params): params['url'] = search_url.format(query=urlencode({'q': query}), api_key=api_key) params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0] return params
def request(query, params): offset = (params['pageno'] - 1) * number_of_results language = params['language'].replace('-', '_').lower() if language.split('-')[0] != 'zh': language = language.split('-')[0] if params['safesearch'] >= 1: safesearch = 1 else: safesearch = 0 # rxieu is some kind of hash from the search query, but accepts random atm search_path = search_string.format(query=urlencode({'q': query}), offset=offset, number_of_results=number_of_results, rxikd=int(time() * 1000), rxieu=random.randint(1000000000, 9999999999), ulse=random.randint(100000000, 999999999), lang=language, safesearch=safesearch) params['url'] = base_url + search_path return params
def request(query, params): offset = (params['pageno'] - 1) * 10 language = match_language(params['language'], supported_languages) language_array = language.split('-') if params['language'].find('-') > 0: country = params['language'].split('-')[1] elif len(language_array) == 2: country = language_array[1] else: country = 'US' url_lang = 'lang_' + language if use_locale_domain: google_hostname = country_to_hostname.get(country.upper(), default_hostname) else: google_hostname = default_hostname # original format: ID=3e2b6616cee08557:TM=5556667580:C=r:IP=4.1.12.5-:S=23ASdf0soFgF2d34dfgf-_22JJOmHdfgg params['cookies']['GOOGLE_ABUSE_EXEMPTION'] = 'x' params['url'] = search_url.format(offset=offset, query=urlencode({'q': query}), hostname=google_hostname, lang=url_lang, lang_short=language) if params['time_range'] in time_range_dict: params['url'] += time_range_search.format(range=time_range_dict[params['time_range']]) params['headers']['Accept-Language'] = language + ',' + language + '-' + country params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' params['google_hostname'] = google_hostname return params
def request(query, params): offset = (params['pageno'] - 1) * number_of_results string_args = dict(query=urlencode({'srsearch': query}), offset=offset, limit=number_of_results, searchtype=search_type) format_strings = list(Formatter().parse(base_url)) if params['language'] == 'all': language = 'en' else: language = params['language'].split('-')[0] # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)] if any(x[1] == 'language' for x in format_strings): string_args['language'] = language # write search-language back to params, required in response params['language'] = language search_url = base_url + search_postfix params['url'] = search_url.format(**string_args) return params
def request(query, params): offset = (params['pageno'] - 1) * 10 params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset) return params
def duckduckgo(query, lang): # duckduckgo autocompleter url = 'https://ac.duckduckgo.com/ac/?{0}&type=list' resp = loads(get(url.format(urlencode(dict(q=query)))).text) if len(resp) > 1: return resp[1] return []
def startpage(query, lang): # startpage autocompleter url = 'https://startpage.com/do/suggest?{query}' resp = get(url.format(query=urlencode({'query': query}))).text.split('\n') if len(resp) > 1: return resp return []
def wikipedia(query, lang): # wikipedia autocompleter url = 'https://' + lang + '.wikipedia.org/w/api.php?action=opensearch&{0}&limit=10&namespace=0&format=json' resp = loads(get(url.format(urlencode(dict(search=query)))).text) if len(resp) > 1: return resp[1] return []
def request(query, params): locale = match_language(params['language'], supported_languages) params['url'] = search_url.format( query=urlencode({'search': query, 'localization': locale}), pageno=params['pageno']) return params
def request(query, params): if query.islower(): query = u'{0}|{1}'.format(query.decode('utf-8'), query.decode('utf-8').title()).encode('utf-8') params['url'] = search_url.format(query=urlencode({'titles': query}), language=url_lang(params['language'])) return params
def get_vqd(query, headers): query_url = site_url.format(query=urlencode({'q': query})) res = get(query_url, headers=headers) content = res.text if content.find('vqd=\'') == -1: raise Exception('Request failed') vqd = content[content.find('vqd=\'') + 5:] vqd = vqd[:vqd.find('\'')] return vqd
def request(query, params): params['url'] = search_url.format( category=type_map[params['category']], q=urlencode({ 'q': query, 'page': params['pageno'] }) ) return params
def request(query, params): if params['time_range'] and params['time_range'] not in time_range_dict: return params offset = (params['pageno'] - 1) * 30 region_code = get_region_code(params['language'], supported_languages) if region_code: params['url'] = url.format( query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset) else: params['url'] = url.format( query=urlencode({'q': query}), offset=offset, dc_param=offset) if params['time_range'] in time_range_dict: params['url'] += time_range_url.format(range=time_range_dict[params['time_range']]) return params
def request(query, params): offset = (params['pageno'] - 1) * 10 if categories[0] and categories[0] in category_to_keyword: params['url'] = url.format(keyword=category_to_keyword[categories[0]], query=urlencode({'q': query}), offset=offset) else: params['url'] = url.format(keyword='web', query=urlencode({'q': query}), offset=offset) # add language tag language = match_language(params['language'], supported_languages) params['url'] += '&locale=' + language.replace('-', '_').lower() return params
def request(query, params): params['url'] = search_url.format(query=urlencode({'q': query}), api_key=api_key) # add language tag if specified if params['language'] != 'all': params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0] return params
def request(query, params): params['url'] = search_url + urlencode({'q': query}) # set language if specified if params['language'] != 'all': params['cookies']['lang'] = params['language'].split('-')[0] else: params['cookies']['lang'] = 'en' return params
def clean_url(url): parsed = urlparse(url) query = [(k, v) for (k, v) in parse_qsl(parsed.query) if k not in ['ixid', 's']] return urlunparse((parsed.scheme, parsed.netloc, parsed.path, parsed.params, urlencode(query), parsed.fragment))
def request(query, params): search_options = { 'start': (params['pageno'] - 1) * number_of_results } if params['time_range'] in time_range_dict: search_options['tbs'] = time_range_attr.format(range=time_range_dict[params['time_range']]) if safesearch and params['safesearch']: search_options['safe'] = 'on' params['url'] = search_url.format(query=urlencode({'q': query}), search_options=urlencode(search_options)) language = match_language(params['language'], supported_languages).split('-')[0] if language: params['url'] += '&lr=lang_' + language return params
def request(query, params): offset = (params['pageno'] - 1) * 10 if categories[0] and categories[0] in category_to_keyword: params['url'] = url.format(keyword=category_to_keyword[categories[0]], query=urlencode({'q': query}), offset=offset) else: params['url'] = url.format(keyword='web', query=urlencode({'q': query}), offset=offset) # add language tag if params['language'] != 'all': language = match_language(params['language'], supported_languages, language_aliases) params['url'] += '&locale=' + language.replace('-', '_').lower() return params
def request(query, params): # to avoid running actual external requests when testing if 'is_test' not in params: vqd = get_vqd(query, params['headers']) else: vqd = '12345' offset = (params['pageno'] - 1) * 50 safesearch = params['safesearch'] - 1 region_code = get_region_code(params['language'], lang_list=supported_languages) if region_code: params['url'] = images_url.format( query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd) else: params['url'] = images_url.format( query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd) return params
def request(query, params): # to avoid running actual external requests when testing if 'is_test' not in params: vqd = get_vqd(query) else: vqd = '12345' offset = (params['pageno'] - 1) * 50 safesearch = params['safesearch'] - 1 region_code = get_region_code(params['language'], lang_list=supported_languages) if region_code: params['url'] = images_url.format( query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd) else: params['url'] = images_url.format( query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd) return params
def request(query, params): offset = (params['pageno'] - 1) * 10 if categories[0] and categories[0] in category_to_keyword: params['url'] = url.format(keyword=category_to_keyword[categories[0]], query=urlencode({'q': query}), offset=offset) else: params['url'] = url.format(keyword='web', query=urlencode({'q': query}), offset=offset) # add language tag if params['language'] != 'all': language = match_language(params['language'], supported_languages, language_aliases) params['url'] += '&locale=' + language.replace('-', '_').lower() params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0' return params
def request(query, params): pageno = (params["pageno"] - 1) * 15 search_url = base_url + "api/v1/search/videos/?pageno={pageno}&{query}" query_dict = {"search": query} language = params["language"].split("-")[0] # pylint: disable=undefined-variable if "all" != language and language in supported_languages: query_dict["languageOneOf"] = language params["url"] = search_url.format(query=urlencode(query_dict), pageno=pageno) return params
def request(query, params): query = urlencode({'q': query})[2:] fp = {'query': query} if paging and search_url.find('{pageno}') >= 0: fp['pageno'] = (params['pageno'] - 1) * page_size + first_page_num params['url'] = search_url.format(**fp) params['query'] = query return params
def request(query, params): # basic search offset = (params['pageno'] - 1) * number_of_results string_args = dict(query=urlencode({'term': query}), offset=offset, hits=number_of_results) params['url'] = base_url.format(**string_args) return params
def request(query, params): if params['time_range'] and params['time_range'] not in time_range_dict: return params params['url'] = search_url.format(page=params['pageno'], query=urlencode({'q': query})) if params['time_range'] in time_range_dict: params['url'] += time_range_url.format( range=time_range_dict[params['time_range']]) return params
def request(query, params): search_options = {'start': (params['pageno'] - 1) * number_of_results} if params['time_range'] in time_range_dict: search_options['tbs'] = time_range_attr.format( range=time_range_dict[params['time_range']]) if safesearch and params['safesearch']: search_options['safe'] = 'on' params['url'] = search_url.format(query=urlencode({'q': query}), search_options=urlencode(search_options)) language = match_language(params['language'], supported_languages).split('-')[0] if language: params['url'] += '&lr=lang_' + language return params
def request(query, params): offset = (params['pageno'] - 1) * 10 + 1 search_path = search_string.format( query=urlencode({'wd': query}), offset=offset) params['url'] = base_url + search_path params['headers']['User-Agent'] = gen_useragent('Windows NT 6.3; WOW64') return params
def request(query, params): if params['language'] == 'all': locale = 'en-US' else: locale = match_language(params['language'], supported_languages) params['url'] = search_url.format( query=urlencode({'search': query, 'localization': locale}), pageno=params['pageno']) return params
def request(query, params): search_options = {'start': (params['pageno'] - 1) * number_of_results} if params['time_range'] in time_range_dict: search_options['tbs'] = time_range_attr.format( range=time_range_dict[params['time_range']]) elif params['time_range'] == 'year': now = date.today() then = now - timedelta(days=365) start = then.strftime('%m/%d/%Y') end = now.strftime('%m/%d/%Y') search_options['tbs'] = time_range_custom_attr.format(start=start, end=end) if safesearch and params['safesearch']: search_options['safe'] = 'active' params['url'] = search_url.format(query=urlencode({'q': query}), search_options=urlencode(search_options)) return params
def request(query, params): search_options = { 'start': (params['pageno'] - 1) * number_of_results } if params['time_range'] in time_range_dict: search_options['tbs'] = time_range_attr.format(range=time_range_dict[params['time_range']]) elif params['time_range'] == 'year': now = date.today() then = now - timedelta(days=365) start = then.strftime('%m/%d/%Y') end = now.strftime('%m/%d/%Y') search_options['tbs'] = time_range_custom_attr.format(start=start, end=end) if safesearch and params['safesearch']: search_options['safe'] = 'active' params['url'] = search_url.format(query=urlencode({'q': query}), search_options=urlencode(search_options)) return params
def request(query, params): if params['time_range'] and params['time_range'] not in time_range_dict: return params offset = (params['pageno'] - 1) * 24 params['url'] = search_url.format(offset=offset, query=urlencode({'q': query})) if params['time_range'] in time_range_dict: params['url'] += time_range_url.format(range=time_range_dict[params['time_range']]) return params
def request(query, params): """Google-Video search request""" language, country, lang_country = get_lang_country( # pylint: disable=undefined-variable params, supported_languages, language_aliases) subdomain = 'www.' + google_domains.get(country.upper(), 'google.com') query_url = 'https://' + subdomain + '/search' + "?" + urlencode( { 'q': query, 'tbm': "isch", 'hl': lang_country, 'lr': "lang_" + language, 'ie': "utf8", 'oe': "utf8", 'num': 30, }) if params['time_range'] in time_range_dict: query_url += '&' + urlencode( {'tbs': 'qdr:' + time_range_dict[params['time_range']]}) if params['safesearch']: query_url += '&' + urlencode( {'safe': filter_mapping[params['safesearch']]}) params['url'] = query_url logger.debug("query_url --> %s", query_url) params['headers']['Accept-Language'] = ("%s,%s;q=0.8,%s;q=0.5" % (lang_country, language, language)) logger.debug("HTTP Accept-Language --> %s", params['headers']['Accept-Language']) params['headers']['Accept'] = ( 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' ) # params['google_subdomain'] = subdomain return params
def request(query, params): if params['time_range'] not in ( None, 'None', '') and params['time_range'] not in time_range_dict: return params offset = (params['pageno'] - 1) * 30 region_code = get_region_code(params['language'], supported_languages) params['url'] = 'https://duckduckgo.com/html/' if params['pageno'] > 1: params['method'] = 'POST' params['data']['q'] = query params['data']['s'] = offset params['data']['dc'] = 30 params['data']['nextParams'] = '' params['data']['v'] = 'l' params['data']['o'] = 'json' params['data']['api'] = '/d.js' if params['time_range'] in time_range_dict: params['data']['df'] = time_range_dict[params['time_range']] if region_code: params['data']['kl'] = region_code else: if region_code: params['url'] = url.format(query=urlencode({ 'q': query, 'kl': region_code }), offset=offset, dc_param=offset) else: params['url'] = url.format(query=urlencode({'q': query}), offset=offset, dc_param=offset) if params['time_range'] in time_range_dict: params['url'] += time_range_url.format( range=time_range_dict[params['time_range']]) return params
def request(query, params): offset = (params['pageno'] - 1) * number_of_results search_type = search_types.get(params.get('category'), '0') params['url'] = base_url +\ search_url.format(query=urlencode({'query': query}), offset=offset, limit=number_of_results, search_type=search_type) params['url'] += '&lr=lang_' + params['language'].split('-')[0] return params
def request(query, params): offset = (params['pageno'] - 1) * 10 + 1 lang = params['language'].split('-')[0].upper() query = u'language:{} {}'.format(lang, query.decode('utf-8')).encode('utf-8') search_path = search_string.format(query=urlencode({'q': query}), offset=offset) params['url'] = base_url + search_path return params
def request(query, params): params['url'] = base_url +\ search_string.format(query=urlencode({'q': query}), limit=number_of_results) language = params['language'].split('-')[0] if language in supported_languages: params['url'] = params['url'] + "&lang=" + language # using searx User-Agent params['headers']['User-Agent'] = searx_useragent() return params
def google(query, lang): # google autocompleter autocomplete_url = 'https://suggestqueries.google.com/complete/search?client=toolbar&' response = get(autocomplete_url + urlencode(dict(hl=lang, q=query))) results = [] if response.ok: dom = etree.fromstring(response.text) results = dom.xpath('//suggestion/@data') return results
def get_as_url_params(self): settings_kv = {} for k, v in self.key_value_settings.items(): if isinstance(v, MultipleChoiceSetting): settings_kv[k] = ','.join(v.get_value()) else: settings_kv[k] = v.get_value() settings_kv['disabled_engines'] = ','.join(self.engines.disabled) settings_kv['enabled_engines'] = ','.join(self.engines.enabled) return urlsafe_b64encode( compress(urlencode(settings_kv).encode('utf-8'))).decode('utf-8')
def request(query, params): offset = (params['pageno'] - 1) * 10 + 1 lang = match_language(params['language'], supported_languages, language_aliases) query = u'language:{} {}'.format(lang.split('-')[0].upper(), query.decode('utf-8')).encode('utf-8') search_path = search_string.format( query=urlencode({'q': query}), offset=offset) params['url'] = base_url + search_path return params
def dbpedia(query, lang): # dbpedia autocompleter, no HTTPS autocomplete_url = 'http://lookup.dbpedia.org/api/search.asmx/KeywordSearch?' response = get(autocomplete_url + urlencode(dict(QueryString=query))) results = [] if response.ok: dom = etree.fromstring(response.content) results = dom.xpath('//a:Result/a:Label//text()', namespaces={'a': 'http://lookup.dbpedia.org/'}) return results
def request(query, params): '''pre-request callback params<dict>: method : POST/GET headers : {} data : {} # if method == POST url : '' category: 'search category' pageno : 1 # number of the requested page ''' search = urlencode({'term': query}) params['url'] = f'https://memucho.de/api/edusharing/search?{search}' return params
def request(query, params): offset = (params['pageno'] - 1) * 10 + 1 language = match_language(params['language'], supported_languages, language_aliases).split('-')[0] params['url'] = search_url.format(offset=offset, query=urlencode({'p': query}), lang=language) # TODO required? params['cookies']['sB'] = '"v=1&vm=p&fl=1&vl=lang_{lang}&sh=1&pn=10&rw=new'\ .format(lang=language) return params
def request(query, params): offset = (params['pageno'] - 1) * 10 if params['language'] == 'all' or params['language'] == 'en-US': language = 'en-GB' else: language = match_language(params['language'], supported_languages, language_aliases) language_array = language.split('-') if params['language'].find('-') > 0: country = params['language'].split('-')[1] elif len(language_array) == 2: country = language_array[1] else: country = 'US' url_lang = 'lang_' + language if use_locale_domain: google_hostname = country_to_hostname.get(country.upper(), default_hostname) else: google_hostname = default_hostname # original format: ID=3e2b6616cee08557:TM=5556667580:C=r:IP=4.1.12.5-:S=23ASdf0soFgF2d34dfgf-_22JJOmHdfgg params['cookies']['GOOGLE_ABUSE_EXEMPTION'] = 'x' params['url'] = search_url.format(offset=offset, query=urlencode({'q': query}), hostname=google_hostname, lang=url_lang, lang_short=language) if params['time_range'] in time_range_dict: params['url'] += time_range_search.format( range=time_range_dict[params['time_range']]) params['headers'][ 'Accept-Language'] = language + ',' + language + '-' + country params['headers'][ 'Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' # Force Safari 3.1 on Mac OS X (Leopard) user agent to avoid loading the new UI that Searx can't parse params['headers']['User-Agent'] = ( "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4)" "AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1") params['google_hostname'] = google_hostname return params
def request(query, params): if params['time_range'] and params['time_range'] not in time_range_dict: return params offset = (params['pageno'] - 1) * 30 region_code = get_region_code(params['language']) if region_code: params['url'] = url.format(query=urlencode({ 'q': query, 'kl': region_code }), offset=offset, dc_param=offset) else: params['url'] = url.format(query=urlencode({'q': query}), offset=offset, dc_param=offset) if params['time_range'] in time_range_dict: params['url'] += time_range_url.format( range=time_range_dict[params['time_range']]) return params
def response(resp): results = [] html = fromstring(resp.text) wikidata_ids = html.xpath(wikidata_ids_xpath) language = match_language(resp.search_params['language'], supported_languages).split('-')[0] # TODO: make requests asynchronous to avoid timeout when result_count > 1 for wikidata_id in wikidata_ids[:result_count]: url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language})) htmlresponse = get(url) jsonresponse = loads(htmlresponse.text) results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language']) return results
def get_as_url_params(self): settings_kv = {} for k, v in self.key_value_settings.items(): if isinstance(v, MultipleChoiceSetting): settings_kv[k] = ','.join(v.get_value()) else: settings_kv[k] = v.get_value() settings_kv['disabled_engines'] = ','.join(self.engines.disabled) settings_kv['enabled_engines'] = ','.join(self.engines.enabled) settings_kv['disabled_plugins'] = ','.join(self.plugins.disabled) settings_kv['enabled_plugins'] = ','.join(self.plugins.enabled) return urlencode(settings_kv)
def image_proxify(url): if url.startswith('//'): url = 'https:' + url if not request.preferences.get_value('image_proxy'): return url if settings.get('result_proxy'): return proxify(url) h = new_hmac(settings['server']['secret_key'], url.encode('utf-8')) return '{0}?{1}'.format(url_for('image_proxy'), urlencode(dict(url=url.encode('utf-8'), h=h)))
def qwant(query, lang): # qwant autocompleter (additional parameter : lang=en_en&count=xxx ) url = 'https://api.qwant.com/api/suggest?{query}' resp = get(url.format(query=urlencode({'q': query, 'lang': lang}))) results = [] if resp.ok: data = loads(resp.text) if data['status'] == 'success': for item in data['data']['items']: results.append(item['value']) return results
def request(query, params): # replace shortcuts with API advanced search keywords for key in shorcut_dict.keys(): query = re.sub(str(key), str(shorcut_dict[key]), query) # basic search offset = (params['pageno'] - 1) * number_of_results string_args = dict(query=urlencode({'query': query}), offset=offset, hits=number_of_results) params['url'] = base_url.format(**string_args) params['headers']['User-Agent'] = searx_useragent() return params
def proxify(url): if url.startswith('//'): url = 'https:' + url if not settings.get('result_proxy'): return url url_params = dict(mortyurl=url.encode('utf-8')) if settings['result_proxy'].get('key'): url_params['mortyhash'] = hmac.new(settings['result_proxy']['key'], url.encode('utf-8'), hashlib.sha256).hexdigest() return '{0}?{1}'.format(settings['result_proxy']['url'], urlencode(url_params))
def request(query, params): offset = (params['pageno'] - 1) * 20 params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset) r = requests.post( 'https://accounts.spotify.com/api/token', data={'grant_type': 'client_credentials'}, headers={'Authorization': 'Basic ' + base64.b64encode( "{}:{}".format(api_client_id, api_client_secret).encode('utf-8') ).decode('utf-8')} ) j = loads(r.text) params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))} return params