def work():
global total_retrieved_pages
while True:
print("retrieved pages: " + str(total_retrieved_pages))
url = queue.get()
if (total_retrieved_pages < max_retrieved_pages):
Spider.crawl_page(threading.current_thread().name, url)
else:
Spider.crawl_page_graph(threading.current_thread().name, url)
print("Crawling task by thread is done")
queue.task_done()
'''
