def request(query, params): search_type = search_types.get(params["category"], "0") params["url"] = search_url.format(search_term=quote(query), search_type=search_type) return params
def response(resp): results = [] search_results = loads(resp.text) # return empty array if there are no results if not search_results.get('query', {}).get('search'): return [] # parse results for result in search_results['query']['search']: #if result.get('snippet', '').startswith('#REDIRECT'): # continue #url = base_url.format(language=resp.search_params['language']) +\ url = base_url + 'wiki/' + quote(result['title'].replace( ' ', '_').encode('utf-8')) # append result results.append({ 'url': url, 'title': result['title'], # 'content': ''}) 'content': result['snippet'] }) # return results return results
def request(query, params): search_type = search_types.get(params['category'], '0') params['url'] = search_url.format(search_term=quote(query), search_type=search_type, pageno=params['pageno'] - 1) return params
def request(query, params): search_type = search_types.get(params['category'], '0') params['url'] = search_url.format(search_term=quote(query), search_type=search_type, pageno=params['pageno'] - 1) return params
def request(query, params): '''pre-request callback params<dict>: method : POST/GET headers : {} data : {} # if method == POST url : '' category: 'search category' pageno : 1 # number of the requested page ''' offset = (params['pageno'] - 1) if offset == 0: search_url_fmt = base_url + 'suchen/dudenonline/{query}' params['url'] = search_url_fmt.format(query=quote(query)) else: params['url'] = search_url.format(offset=offset, query=quote(query)) return params
def request(query, params): if query.islower(): query = query.title() params['url'] = search_url.format(title=quote(query), language=url_lang(params['language'])) params['headers']['User-Agent'] = searx_useragent() return params
def response(resp): results = [] dom = html.fromstring(resp.text) search_res = dom.xpath('//table[@class="data"]//tr') # return empty array if nothing is found if not search_res: return [] # parse results for result in search_res[1:]: link = result.xpath('.//a[@class="cellMainLink"]')[0] href = urljoin(url, link.attrib['href']) title = extract_text(link) content = extract_text(result.xpath(content_xpath)) seed = extract_text(result.xpath('.//td[contains(@class, "green")]')) leech = extract_text(result.xpath('.//td[contains(@class, "red")]')) filesize_info = extract_text( result.xpath('.//td[contains(@class, "nobr")]')) files = extract_text( result.xpath('.//td[contains(@class, "center")][2]')) seed = convert_str_to_int(seed) leech = convert_str_to_int(leech) filesize, filesize_multiplier = filesize_info.split() filesize = get_torrent_size(filesize, filesize_multiplier) if files.isdigit(): files = int(files) else: files = None magnetlink = result.xpath(magnet_xpath)[0].attrib['href'] torrentfile = result.xpath(torrent_xpath)[0].attrib['href'] torrentfileurl = quote(torrentfile, safe="%/:=&?~#+!$,;'@()*") # append result results.append({ 'url': href, 'title': title, 'content': content, 'seed': seed, 'leech': leech, 'filesize': filesize, 'files': files, 'magnetlink': magnetlink, 'torrentfile': torrentfileurl, 'template': 'torrent.html' }) # return results sorted by seeder return sorted(results, key=itemgetter('seed'), reverse=True)
def request(query, params): search_type = search_types.get(params['category'], 'all') pageno = (params['pageno'] - 1) * 50 params['url'] = search_url.format(search_term=quote(query), search_type=search_type, pageno=pageno) params['cookies'] = cookies return params
def response(resp): results = [] dom = html.fromstring(resp.text) search_res = dom.xpath('//table[@class="data"]//tr') # return empty array if nothing is found if not search_res: return [] # parse results for result in search_res[1:]: link = result.xpath('.//a[@class="cellMainLink"]')[0] href = urljoin(url, link.attrib['href']) title = extract_text(link) content = extract_text(result.xpath(content_xpath)) seed = extract_text(result.xpath('.//td[contains(@class, "green")]')) leech = extract_text(result.xpath('.//td[contains(@class, "red")]')) filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]')) files = extract_text(result.xpath('.//td[contains(@class, "center")][2]')) seed = convert_str_to_int(seed) leech = convert_str_to_int(leech) filesize, filesize_multiplier = filesize_info.split() filesize = get_torrent_size(filesize, filesize_multiplier) if files.isdigit(): files = int(files) else: files = None magnetlink = result.xpath(magnet_xpath)[0].attrib['href'] torrentfile = result.xpath(torrent_xpath)[0].attrib['href'] torrentfileurl = quote(torrentfile, safe="%/:=&?~#+!$,;'@()*") # append result results.append({'url': href, 'title': title, 'content': content, 'seed': seed, 'leech': leech, 'filesize': filesize, 'files': files, 'magnetlink': magnetlink, 'torrentfile': torrentfileurl, 'template': 'torrent.html'}) # return results sorted by seeder return sorted(results, key=itemgetter('seed'), reverse=True)
def request(query, params): '''pre-request callback params<dict>: method : POST/GET headers : {} data : {} # if method == POST url : '' category: 'search category' pageno : 1 # number of the requested page ''' offset = (params['pageno'] - 1) params['url'] = search_url.format(offset=offset, query=quote(query)) return params
def response(resp): results = [] search_result = loads(resp.text) # wikipedia article's unique id # first valid id is assumed to be the requested article if 'pages' not in search_result['query']: return results for article_id in search_result['query']['pages']: page = search_result['query']['pages'][article_id] if int(article_id) > 0: break if int(article_id) < 0 or 'disambiguation' in page.get('pageprops', {}): return [] title = page.get('title') image = page.get('thumbnail') if image: image = image.get('source') extract = page.get('extract') summary = extract_first_paragraph(extract, title, image) summary = summary.replace('() ', '') # link to wikipedia article wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \ + 'wiki/' + quote(title.replace(' ', '_').encode('utf8')) results.append({'url': wikipedia_link, 'title': title}) results.append({ 'infobox': title, 'id': wikipedia_link, 'content': summary, 'img_src': image, 'urls': [{ 'title': 'Wikipedia', 'url': wikipedia_link }] }) return results
def response(resp): results = [] search_result = loads(resp.text) # wikipedia article's unique id # first valid id is assumed to be the requested article for article_id in search_result['query']['pages']: page = search_result['query']['pages'][article_id] if int(article_id) > 0: break if int(article_id) < 0: return [] title = page.get('title') image = page.get('thumbnail') if image: image = image.get('source') extract = page.get('extract') summary = extract_first_paragraph(extract, title, image) # link to wikipedia article wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \ + 'wiki/' + quote(title.replace(' ', '_').encode('utf8')) results.append({'url': wikipedia_link, 'title': title}) results.append({'infobox': title, 'id': wikipedia_link, 'content': summary, 'img_src': image, 'urls': [{'title': 'Wikipedia', 'url': wikipedia_link}]}) return results
def response(resp): results = [] search_results = loads(resp.text) # return empty array if there are no results if not search_results.get('query', {}).get('search'): return [] # parse results for result in search_results['query']['search']: if result.get('snippet', '').startswith('#REDIRECT'): continue url = base_url.format(language=resp.search_params['language']) +\ 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8')) # append result results.append({'url': url, 'title': result['title'], 'content': ''}) # return results return results
def request(query, params): params['url'] = search_url.format(search_term=quote(query), pageno=params['pageno'] - 1) return params
def request(query, params): search_path = search_string.format(query=quote(query), ) params['url'] = base_url + search_path return params
def request(query, params): params['url'] = search_url.format(search_term=quote(query), page_no=params['pageno'] - 1) return params
def request(query, params): params['url'] = search_url.format(channel=channel, page=params['pageno'], query=quote(query)) return params