pass
# parsing finished
for raw_link in soup.find_all('a'):
filtered_addtolist(raw_link.get('href'), link_queue, link_visited)
page = Webpage(target_url, byte_title, date, byte_content)
date_filename = space_to_dash(str(date))
database.save(date_filename, byte_content)
print(date_filename)
return link_queue, link_visited
if __name__ == '__main__':
while True:
link_queue = QueueManager.list_init()
link_visited = isVisited.init()
for iter_count in tqdm(range(MAX_ITERATION)):
link_queue, link_visited\
= parse_an_article(link_queue, link_visited)
print("now start saving")
QueueManager.list_save(link_queue)
isVisited.save(link_visited)
print("saving complete")
while True:
user_input = input("continue?\n(y/n)")
if user_input == 'y':
break
if user_input == 'n':
exit(0)
