def work(): while True: url = queue.get() Spider.crawl_page(threading.current_thread().name, url) queue.task_done()
def work(): while True: url = queue.get() Spider.crawl_page(threading.current_thread().name, url) queue.task_done()
def work(): while True: url = queue.get() Spider.crawl_page(threading.current_thread().name, url) queue.task_done( ) # Notifies main module that job is done. helps free up memory
def assign_url_to_thread(): while True: url = queue.get() Spider.crawl_page(url) queue.task_done()