def url_manager_proc(self, url_q, conn_q, root_url):
url_manager = UrlManager()
url_manager.add_new_url(root_url)
while True:
while url_manager.has_new_url():
# 从URL管理器获取新的URL
new_url = url_manager.get_new_url()
# 将新的URL发给工作节点
url_q.put(new_url)
print('old_url=', url_manager.old_url_size())
# 加一个判断条件,当爬取2000个链接后就关闭，并保存进度
if url_manager.old_url_size() > 2000:
# 通知爬行节点工作结束
url_q.put('end')
print('控制节点发起结束通知!')
# 关闭管理节点，同时存储set状态
url_manager.save_progress('new_urls.txt',
url_manager.new_urls)
url_manager.save_progress('old_urls.txt',
url_manager.old_urls)
return
# 将从result_solve_proc获取到的URL添加到URL管理器
try:
if not conn_q.empty():
urls = conn_q.get()
url_manager.add_new_url(urls)
except BaseException:
# 延时休息
time.sleep(0.1)
count += 1
new_url = url.get_new_url()
print('正在爬第' + str(count) + '条：' + new_url)
mutex.release()
html = downloader.download(new_url)
url_list = parser.parser(html)
url.add_new_urls(url_list)
except:
print('未知异常')
if __name__ == '__main__':
# 添加新的URL,并开始一轮爬取
url.add_new_url(
'https://baike.baidu.com/item/%E6%99%BA%E6%85%A7%E5%9C%B0%E7%90%83/1071533'
)
crawl()
# 添加旧的URL,在进行了一轮爬取后再添加旧的
print('已经添加旧的url')
all_url_path = './all_url.txt'
all_url_file = open(all_url_path, mode='r', encoding='utf-8')
url_list = all_url_file.readlines()
for url_1 in url_list:
if url_1.find('http') == 0:
url.add_old_url(url_1[:-1])
# 开启多线程
while url.has_new_url():
t = None
