def url_manager_proc(url_q, conn_q, root_url, num=6):
"""
:param url_q:里面放的是url集合单个url
:param conn_q:里面放的是url集合
:param root_url:
:param num:
:return:
"""
url_manager = UrlManager()
url_manager.add_new_url(root_url)
while True:
while url_manager.has_new_url():
print("# url_manager_proc将要爬取的url放入url_q中")
new_url = url_manager.get_new_url()
print(new_url)
url_q.put(new_url)
if url_manager.old_url_size() > num:
# 通知爬行节点工作结束
url_q.put('end')
print('控制节点发起结束通知!')
# 关闭管理节点，同时存储 set 状态
url_manager.save_progress()
break
try:
if not conn_q.empty():
print("# url_manager_proc从conn_q中拿取urls")
urls = conn_q.get()
print(urls)
url_manager.add_new_urls(urls)
else:
# 延时休息
time.sleep(0.1)
except Exception as e:
print(e)
def url_manager_proc(self, url_que, conn_que, root_url):
url_manager = UrlManager()
url_manager.add_new_url(root_url)
while True:
while url_manager.has_new_url():
new_url = url_manager.get_new_url()
# 将新的url发给工作节点
url_que.put(new_url)
print('old_url=', url_manager.old_urls_size())
if url_manager.old_urls_size() > 2000:
url_que.put('end')
print('控制节点发出结束通知')
# 关闭管理节点，同时存储set状态
url_manager.save_progress('new_urls.txt',
url_manager.new_urls)
url_manager.save_progress('old_urls.txt',
url_manager.old_urls)
return
# 将从result_solve_proc 获取的urls添加到URL管理器
try:
if not conn_que.empty():
urls = conn_que.get()
for url in urls:
url_manager.add_new_url(url)
except BaseException:
time.sleep(0.1)
def url_manager_proc(self,url_q,conn_q,root_url):
url_manager = UrlManager()
url_manager.add_new_url(root_url)
while True:
while(url_manager.has_new_url()):
new_url = url_manager.get_new_url()
url_q.put(new_url)
print('old_url=',url_manager.old_url_size())
if(url_manager.old_url_size()>30):
url_q.put('end')
print('控制节点发起结束通知!')
url_manager.save_progress('new_urls.txt',url_manager.new_urls)
url_manager.save_progress('old_urls.txt',url_manager.old_urls)
return
try:
urls = conn_q.get()
url_manager.add_new_urls(urls)
except:
time.sleep(0.1)
def url_manager_proc(self, url_q, conn_q, root_url):
url_manager = UrlManager()
url_manager.add_new_url(root_url)
while True:
while (url_manager.has_new_url()):
new_url = url_manager.get_new_url()
url_q.put(new_url)
print('old_url=', url_manager.old_url_size())
if (url_manager.old_url_size() > 2000):
url_q.put('end')
print('控制节点发起通知')
url_manager.save_progress('new_urls.txt',
url_manager.new_urls)
url_manager.save_progress('old_urls.txt',
url_manager.old_urls)
return
try:
if not conn_q.empty():
urls = conn_q.get()
url_manager.add_new_urls()
except BaseException as e:
print(e)
time.sleep(0.1)
def url_manager_proc(self, url_q, conn_q, root_url, num=200):
url_manager = UrlManager()
url_manager.add_new_url(root_url)
while True:
while url_manager.has_new_url():
new_url = url_manager.get_new_url()
url_q.put(new_url)
if url_manager.old_url_size() > num:
# 通知爬行节点工作结束
url_q.put('end')
print('控制节点发起结束通知!')
# 关闭管理节点，同时存储 set 状态
url_manager.save_progress()
return
# 没有url了就从conn_q里拿
try:
if not conn_q.empty():
urls = conn_q.get()
url_manager.add_new_urls(urls)
else:
# 延时休息
time.sleep(0.1)
except Exception as e:
print(e)
def url_manager_proc(self, url_q, conn_q, root_url):
url_manager = UrlManager()
url_manager.add_new_url(root_url)
while True:
while url_manager.has_new_url():
new_url = url_manager.get_new_url()
print("url " + new_url)
url_q.put(new_url)
# print("old_url=",url_manager.old_url_size())
if url_manager.old_url_size() > 2000:
url_q.put("end")
print("控制节点发起结束通知!")
url_manager.save_progress("new_urls.txt",
url_manager.new_urls)
url_manager.save_progress("old_urls.txt",
url_manager.old_urls)
return
try:
if not conn_q.empty():
urls = conn_q.get()
# print(urls)
url_manager.add_new_urls(urls)
except BaseException:
time.sleep(0.1)
import pickle
import hashlib
print("has_new_url", UrlManager.has_new_url.__doc__)
print("add_new_url", UrlManager.add_new_url.__doc__)
print("add_new_urls", UrlManager.add_new_urls.__doc__)
print("get_new_url", UrlManager.get_new_url.__doc__)
print("new_url_size", UrlManager.new_url_size.__doc__)
print("old_url_size", UrlManager.old_url_size.__doc__)
print("save_progress", UrlManager.save_progress.__doc__)
print("load_progress", UrlManager.load_progress.__doc__)
urls = set([
"http://qq.ip138.com/tianqi/", "http://qq.ip138.com/shenfenzheng/",
"http://qq.ip138.com/huoche/",
"http://qq.ip138.com/daishoudian/mobile.htm",
"http://www.miitbeian.gov.cn/"
])
urlmanager = UrlManager()
print(type(urls))
# urlmanager获得新的url集合
urlmanager.add_new_urls(urls)
print(urlmanager.has_new_url())
# urlmanager输出一个未爬取的url
new_url = urlmanager.get_new_url() #拿出的同时将其放的到已经爬取的url集合中
# 没有未爬取的url时返回None
print(new_url)
print(urlmanager.old_url_size())
# 保存进度
urlmanager.save_progress()