def add_links_to_queue(links, distance):
for url in links:
if Database.check_if_in_visited(url) is True:
continue
if Database.check_if_in_unvisited(url) is True:
continue
Database.add_url_to_unvisited(url, distance)
def add_links_to_queue(links, distance):
for url in links:
if Database.check_if_in_visited(url) is True:
continue
if Database.check_if_in_unvisited(url) is True:
continue
Database.add_url_to_unvisited(url, distance)
def create_jobs():
for item in Connection.collection_unvisited.find():
url = item['url']
distance = item['distance']
if Database.check_if_in_visited(url) is False and (domain == get_domain_name(url) or broad_search is True):
thread_queue.put(item)
Database.add_url_to_visited(url, distance)
thread_queue.join()
create_jobs()
def create_jobs():
for item in Connection.collection_unvisited.find():
url = item['url']
distance = item['distance']
if Database.check_if_in_visited(url) is False and (
domain == get_domain_name(url) or broad_search is True):
thread_queue.put(item)
Database.add_url_to_visited(url, distance)
thread_queue.join()
create_jobs()
