def crawl_repost_page(mid, uid):
limit = get_max_repost_page() + 1
first_repost_data = crawl_repost_by_page(mid, 1)
total_page = repost.get_total_page(first_repost_data[0])
repost_datas = first_repost_data[1]
if not repost_datas:
return
root_user, _ = user_get.get_profile(uid)
if total_page < limit:
limit = total_page + 1
for page_num in range(2, limit):
cur_repost_datas = crawl_repost_by_page(mid, page_num)[1]
if cur_repost_datas:
repost_datas.extend(cur_repost_datas)
for index, repost_obj in enumerate(repost_datas):
user_id = IdNames.fetch_uid_by_name(repost_obj.parent_user_name)
if not user_id:
# when it comes to errors, set the args to default(root)
repost_obj.parent_user_id = root_user.uid
repost_obj.parent_user_name = root_user.name
else:
repost_obj.parent_user_id = user_id
repost_datas[index] = repost_obj
weibo_repost.save_reposts(repost_datas)
def crawl_repost_page(mid, uid):
limit = get_max_repost_page() + 1
first_repost_data = crawl_repost_by_page(mid, 1)
wb_data.set_weibo_repost_crawled(mid)
total_page = repost.get_total_page(first_repost_data[0])
repost_datas = first_repost_data[1]
if not repost_datas:
return
root_user = user_get.get_profile(uid)
if total_page < limit:
limit = total_page + 1
# todo 这里需要衡量是否有用网络调用的必要性
for page_num in range(2, limit):
# app.send_task('tasks.comment.crawl_comment_by_page', args=(mid, page_num), queue='comment_page_crawler',
# routing_key='comment_page_info')
cur_repost_datas = crawl_repost_by_page(mid, page_num)[1]
if cur_repost_datas:
repost_datas.extend(cur_repost_datas)
# 补上user_id，方便可视化
for index, repost_obj in enumerate(repost_datas):
user_id = IdNames.fetch_uid_by_name(repost_obj.parent_user_name)
if not user_id:
# 设置成根用户的uid和用户名
repost_obj.parent_user_id = root_user.uid
repost_obj.parent_user_name = root_user.name
else:
repost_obj.parent_user_id = user_id
repost_datas[index] = repost_obj
weibo_repost.save_reposts(repost_datas)
def crawl_repost_page(mid, uid):
limit = get_max_repost_page() + 1
first_repost_data = crawl_repost_by_page(mid, 1)
total_page = repost.get_total_page(first_repost_data[0])
repost_datas = first_repost_data[1]
if not repost_datas:
return
root_user, _ = user_get.get_profile(uid)
if total_page < limit:
limit = total_page + 1
for page_num in range(2, limit):
cur_repost_datas = crawl_repost_by_page(mid, page_num)[1]
if cur_repost_datas:
repost_datas.extend(cur_repost_datas)
for index, repost_obj in enumerate(repost_datas):
user_id = IdNames.fetch_uid_by_name(repost_obj.parent_user_name)
if not user_id:
# when it comes to errors, set the args to default(root)
repost_obj.parent_user_id = root_user.uid
repost_obj.parent_user_name = root_user.name
else:
repost_obj.parent_user_id = user_id
repost_datas[index] = repost_obj
weibo_repost.save_reposts(repost_datas)
def test_get_total_repost_page(self):
"""
测试获取所有转发页数
:return:
"""
from page_parse import repost
with open('tests/repost.html', encoding='utf-8') as f:
html = f.read()
total_page = repost.get_total_page(html)
self.assertEqual(total_page, 1580)
def test_get_total_repost_page(self):
"""
测试获取所有转发页数
:return:
"""
from page_parse import repost
url = TEST_SERVER + 'repost.html'
resp = requests.get(url)
resp.encoding = 'utf-8'
html = resp.text
total_page = repost.get_total_page(html)
self.assertEqual(total_page, 1580)
def test_get_total_repost_page(self):
"""
测试获取所有转发页数
:return:
"""
from page_parse import repost
url = TEST_SERVER + 'repost.html'
resp = requests.get(url)
resp.encoding = 'utf-8'
html = resp.text
total_page = repost.get_total_page(html)
self.assertEqual(total_page, 1580)