async def fetch_from_urls(searx_result: SearxStatisticsResult,
instances: list):
results = OrderedDict()
for instance in instances:
# basic checks
# url may be different because of redirect
url, detail = await fetch_one(instance)
searx_result.update_instance(url, detail)
# output
http_status_code = detail.get('http').get('status_code', '') or ''
searx_version = detail.get('version', '') or ''
timing = detail.get('timing', {}).get('initial') or None
cert_orgname = (detail.get('tls')
or {}).get('certificate',
{}).get('organizationName', '')
error = detail.get('error', '')
if error != '':
icon = '❌'
elif searx_version == '':
icon = '👽'
else:
icon = '🍰'
if timing:
timing = '{:.3f}'.format(timing)
else:
timing = ' '
print('{0:3} {1} {2:20} {3} {4:60} {5:30} {6:50}'.format(
http_status_code, icon, searx_version, timing, url, cert_orgname,
error))
return results
async def get_searx_stats_result_from_repository() -> SearxStatisticsResult:
"""
Fetch searx instances from https://github.com/searx/searx-instances/
"""
searx_stats_result = SearxStatisticsResult(private=False)
searx_instances = load_searx_instances()
for url, instance in searx_instances.items():
url = add_slash(url)
searx_stats_result.update_instance(
url, {
'comments': instance.comments,
'alternativeUrls': copy_dict_slash(instance.additional_urls),
'main': True,
})
for aurl, comment in instance.additional_urls.items():
aurl = add_slash(aurl)
a_aurls = copy_dict_slash(instance.additional_urls)
a_aurls[url] = ''
if aurl in a_aurls:
del a_aurls[aurl]
searx_stats_result.update_instance(aurl, {
'comments': [comment],
'alternativeUrls': a_aurls
})
return searx_stats_result
async def fetch(searx_stats_result: SearxStatisticsResult):
url_to_deleted = []
url_to_update = OrderedDict()
# fetch and store the changes in url_to_deleted and url_to_add
# do not modify the searx_stats_result.instances to avoid
async def fetch_and_store_change(url: str, detail, *_, **__):
if 'version' not in detail:
r_url, r_detail = await fetch_one_display(
url, searx_stats_result.private)
dict_merge(r_detail, detail)
if r_url != url:
# r_url is the URL after following a HTTP redirect
# in this case the searx_stats_result.instances[url] must be deleted.
url_to_deleted.append(url)
url_to_update[r_url] = r_detail
instance_iterator = searx_stats_result.iter_instances(
only_valid=False, valid_or_private=False)
await for_each(instance_iterator, fetch_and_store_change, limit=1)
# apply the changes
for url in url_to_deleted:
del searx_stats_result.instances[url]
for url, detail in url_to_update.items():
searx_stats_result.update_instance(url, detail)
async def get_searx_stats_result_from_list(
instance_urls: list, private: bool) -> SearxStatisticsResult:
"""
Fetch searx instances from instance_urls given parameter.
"""
searx_stats_result = SearxStatisticsResult(private=private)
for url in instance_urls:
url = add_slash(url)
searx_stats_result.update_instance(url, {
'comments': [],
'alternativeUrls': dict()
})
return searx_stats_result