def testGettingLinksVariation(self):
test_pool = pool.ThreadPool(processes=100)
url = "http://bbc.co.uk"
latest = ArchiveOrg.get_url_info(url, min_size=1, limit=-1)[0]
timestamp =""
if isinstance(latest, ArchiveStruct):
timestamp = latest.date_stamp
info = ArchiveOrg.get_domain_urls(url, limit=2000)
res_count = len(info)
broken_res_count = 0
links = []
for item in info:
item.date_stamp = timestamp
links.append(ArchiveOrg.get_archive_link(item))
results = [test_pool.apply_async(func=test_response, args=(x,)) for x in links]
returned = [y.get() for y in results]
for result in returned:
if result == False:
broken_res_count += 1
print("total:", res_count, " broken res:", broken_res_count)
def testGettingLinksVariation(self):
test_pool = pool.ThreadPool(processes=100)
url = "http://bbc.co.uk"
latest = ArchiveOrg.get_url_info(url, min_size=1, limit=-1)[0]
timestamp = ""
if isinstance(latest, ArchiveStruct):
timestamp = latest.date_stamp
info = ArchiveOrg.get_domain_urls(url, limit=2000)
res_count = len(info)
broken_res_count = 0
links = []
for item in info:
item.date_stamp = timestamp
links.append(ArchiveOrg.get_archive_link(item))
results = [
test_pool.apply_async(func=test_response, args=(x, ))
for x in links
]
returned = [y.get() for y in results]
for result in returned:
if result == False:
broken_res_count += 1
print("total:", res_count, " broken res:", broken_res_count)
