def run_deepweb_test():
t = TestClient(service_host="http://0.0.0.0:8080", auth=('admin', 'admin'))
test_files_dict = {
"Hidden Service lists and search engines": "test_link_sites.json",
"Marketplace Financial": "test_finance.json",
"Marketplace Commercial Services": "test_commercial_services.json",
"Blogs and radios": "test_blogs.json",
"Politics": "test_politics.json"
}
for tests_name, tests_file in test_files_dict.items():
print(
"======================================================================"
)
print("Website group: %s from file %s" % (tests_name, tests_file))
print(
"======================================================================"
)
with open(tests_file) as f:
tests_json = json.load(f)
# Get archives for each individual page
for web_name, url in tests_json.items():
print("Website: %s, URL: %s" % (web_name, url))
data = {
'urls': [url],
'name': "DeepWebTest-{}".format(t.test_number),
'headers': {},
'forceTor': True,
}
timing_start = time.time()
t.get_archive(data=data)
timing_end = time.time()
print(" archiving request time: %s" %
(timing_end - timing_start))
