def response(resp):
results = []
json = loads(resp.text)
if results_query:
rs = query(json, results_query)
if not len(rs):
return results
for result in rs[0]:
try:
url = query(result, url_query)[0]
title = query(result, title_query)[0]
except:
continue
try:
content = query(result, content_query)[0]
except:
content = ""
try:
c_size = query(result, package_compressed_size_query)[0]
i_size = query(result, package_installed_size_query)[0]
pkgver = query(result, package_version_query)[0]
except:
c_size = ""
i_size = ""
pkgver = ""
results.append({
"url": to_string(url),
"title": to_string(title),
"content": to_string(content),
"i_size": i_size,
"c_size": c_size,
"version": pkgver,
})
else:
for url, title, content, c_size, i_size, pkgver in zip(
query(json, url_query),
query(json, title_query),
query(json, content_query),
query(json, package_compressed_size_query),
query(json, package_installed_size_query),
query(json, package_version_query),
):
results.append({
"url": to_string(url),
"title": to_string(title),
"content": to_string(content),
"i_size": i_size,
"c_size": c_size,
"version": pkgver,
})
if not suggestion_query:
return results
for suggestion in query(json, suggestion_query):
results.append({"suggestion": suggestion})
return results
def response(resp):
results = []
result_len = 0
dom = html.fromstring(resp.text)
# parse results
for result in eval_xpath(dom, '//div[@class="sa_cc"]'):
link = eval_xpath(result, './/h3/a')[0]
url = link.attrib.get('href')
title = extract_text(link)
content = extract_text(eval_xpath(result, './/p'))
# append result
results.append({'url': url, 'title': title, 'content': content})
# parse results again if nothing is found yet
for result in eval_xpath(dom, '//li[@class="b_algo"]'):
link = eval_xpath(result, './/h2/a')[0]
url = link.attrib.get('href')
title = extract_text(link)
content = extract_text(eval_xpath(result, './/p'))
# append result
results.append({'url': url, 'title': title, 'content': content})
try:
result_len_container = "".join(
eval_xpath(dom, '//span[@class="sb_count"]/text()'))
result_len_container = utils.to_string(result_len_container)
if "-" in result_len_container:
# Remove the part "from-to" for paginated request ...
result_len_container = result_len_container[
result_len_container.find("-") * 2 + 2:]
result_len_container = re.sub('[^0-9]', '', result_len_container)
if len(result_len_container) > 0:
result_len = int(result_len_container)
except Exception as e:
logger.debug('result error :\n%s', e)
pass
if _get_offset_from_pageno(resp.search_params.get("pageno",
0)) > result_len:
return []
results.append({'number_of_results': result_len})
return results
def response(resp):
results = []
json = loads(resp.text)
if results_query:
rs = query(json, results_query)
if not len(rs):
return results
for result in rs[0]:
try:
url = query(result, url_query)[0]
title = query(result, title_query)[0]
except:
continue
try:
content = query(result, content_query)[0]
except:
content = ""
results.append({
'url': to_string(url),
'title': to_string(title),
'content': to_string(content),
})
else:
for url, title, content in zip(
query(json, url_query),
query(json, title_query),
query(json, content_query)
):
results.append({
'url': to_string(url),
'title': to_string(title),
'content': to_string(content),
})
if not suggestion_query:
return results
for suggestion in query(json, suggestion_query):
results.append({'suggestion': suggestion})
return results
def response(resp):
results = []
json = loads(resp.text)
if results_query:
rs = query(json, results_query)
if not len(rs):
return results
for result in rs[0]:
try:
url = query(result, url_query)[0]
title = query(result, title_query)[0]
except:
continue
try:
content = query(result, content_query)[0]
except:
content = ""
results.append({
'url': to_string(url),
'title': to_string(title),
'content': to_string(content),
})
else:
for url, title, content in zip(query(json, url_query),
query(json, title_query),
query(json, content_query)):
results.append({
'url': to_string(url),
'title': to_string(title),
'content': to_string(content),
})
if not suggestion_query:
return results
for suggestion in query(json, suggestion_query):
results.append({'suggestion': suggestion})
return results
def response(resp):
results = []
json = loads(resp.text)
if results_query:
rs = query(json, results_query)
if not len(rs):
return results
for result in rs[0]:
try:
url = query(result, url_query)[0]
title = query(result, title_query)[0]
except:
continue
try:
content = query(result, content_query)[0]
except:
content = ""
try:
popularity = query(result, popularity_query)[0]
quality = query(result, quality_query)[0]
maintenance = query(result, maintenance_query)[0]
except:
maintenance = ""
quality = ""
popularity = ""
results.append({
"url": to_string(url),
"title": to_string(title),
"content": to_string(content),
"scores": {
"popularity": to_string(popularity),
"quality": to_string(quality),
"maintenance": to_string(maintenance),
},
})
else:
for url, title, content, quality, popularity, maintenance in zip(
query(json, url_query),
query(json, title_query),
query(json, content_query),
query(json, quality_query),
query(json, popularity_query),
query(json, maintenance_query),
):
results.append({
"url": to_string(url),
"title": to_string(title),
"content": to_string(content),
"scores": {
"popularity": to_string(popularity),
"quality": to_string(quality),
"maintenance": to_string(maintenance),
},
})
if not suggestion_query:
return results
for suggestion in query(json, suggestion_query):
results.append({"suggestion": suggestion})
return results