def crawl_comment_page(mid):
limit = get_max_comment_page()
cur_page = 1
next_url = ''
while cur_page <= limit:
cur_time = int(time.time()*1000)
if cur_page == 1:
url = start_url.format(mid, cur_time)
else:
url = base_url.format(next_url, cur_time)
html = get_page(url, user_verify=False)
comment_datas = comment.get_comment_list(html, mid)
if not comment_datas and cur_page == 1:
crawler.warning('微博id为{}的微博评论未采集成功，请检查原因'.format(mid))
return
save_comments(comment_datas)
# 由于这里每一步都要根据上一步来迭代，所以不适合采用网络调用（主要是比较麻烦）
next_url = comment.get_next_url(html)
if not next_url:
crawler.info('微博{}的评论采集已经完成'.format(mid))
return
cur_page += 1
def crawl_comment_by_page(mid, page_num):
cur_time = int(time.time() * 1000)
cur_url = base_url.format(mid, page_num, cur_time)
html = get_page(cur_url, user_verify=False)
comment_datas = comment.get_comment_list(html, mid)
save_comments(comment_datas)
wb_data.set_weibo_comment_crawled(mid)
return html
def crawl_comment_by_page(mid, page_num):
cur_url = BASE_URL.format(mid, page_num)
html = get_page(cur_url, auth_level=1, is_ajax=True)
comment_datas = comment.get_comment_list(html, mid)
CommentOper.add_all(comment_datas)
if page_num == 1:
WbDataOper.set_weibo_comment_crawled(mid)
return html, comment_datas
def crawl_comment_by_page(mid, page_num):
cur_time = int(time.time() * 1000)
cur_url = base_url.format(mid, page_num, cur_time)
html = get_page(cur_url, user_verify=False)
comment_datas = comment.get_comment_list(html, mid)
save_comments(comment_datas)
if page_num == 1:
wb_data.set_weibo_comment_crawled(mid)
return html
def test_parse_comment_page(self):
"""
测试对评论页的解析
:return:
"""
from page_parse import comment
with open('tests/comment.html', encoding='utf-8') as f:
html = f.read()
comment_list = comment.get_comment_list(html, '1123331211')
self.assertEqual(len(comment_list), 19)
def test_parse_comment_page(self):
"""
测试对评论页的解析
:return:
"""
from page_parse import comment
url = TEST_SERVER + 'comment.html'
resp = requests.get(url)
resp.encoding = 'utf-8'
html = resp.text
comment_list = comment.get_comment_list(html, '1123331211')
self.assertEqual(len(comment_list), 19)
def test_parse_comment_page(self):
"""
测试对评论页的解析
:return:
"""
from page_parse import comment
url = TEST_SERVER + 'comment.html'
resp = requests.get(url)
resp.encoding = 'utf-8'
html = resp.text
comment_list = comment.get_comment_list(html, '1123331211')
self.assertEqual(len(comment_list), 19)
def crawl_comment_by_page(mid, page_num, seeion):
try:
cur_url = BASE_URL.format(mid, page_num)
html = get_page(cur_url, auth_level=1, is_ajax=True)
comment_datas, seed_ids = comment.get_comment_list(html, mid)
except SoftTimeLimitExceeded:
crawler.error(
"comment SoftTimeLimitExceeded mid={mid} page_num={page_num}".
format(mid=mid, page_num=page_num))
crawl_comment_by_page(mid, page_num)
CommentOper.add_all(comment_datas, seeion)
SeedidsOper.insert_seeds(seed_ids, seeion)
if page_num == 1:
WbDataOper.set_weibo_comment_crawled(mid, seeion)
return html, comment_datas
def crawl_comment_by_page(mid, page_num):
try:
cur_url = BASE_URL.format(mid, page_num)
html = get_page(cur_url, auth_level=1, is_ajax=True)
comment_datas = comment.get_comment_list(html, mid)
except SoftTimeLimitExceeded:
crawler.error(
"comment SoftTimeLimitExceeded mid={mid} page_num={page_num}".
format(mid=mid, page_num=page_num))
app.send_task('tasks.comment.crawl_comment_by_page',
args=(mid, page_num),
queue='comment_page_crawler',
routing_key='comment_page_info')
CommentOper.add_all(comment_datas)
if page_num == 1:
WbDataOper.set_weibo_comment_crawled(mid)
return html, comment_datas
def crawl_comment_by_page(mid, page_num):
try:
cur_url = BASE_URL.format(mid, page_num)
html = get_page(cur_url, auth_level=1, is_ajax=True)
comment_datas = comment.get_comment_list(html, mid)
except SoftTimeLimitExceeded:
crawler.error(
"comment SoftTimeLimitExceeded mid={mid} page_num={page_num}".
format(mid=mid, page_num=page_num))
app.send_task(
'tasks.comment.crawl_comment_by_page',
args=(mid, page_num),
queue='comment_page_crawler',
routing_key='comment_page_info')
CommentOper.add_all(comment_datas)
if page_num == 1:
WbDataOper.set_weibo_comment_crawled(mid)
return html, comment_datas
def test_parse_comment_info(cookies):
url = 'http://weibo.com/aj/v6/comment/big?ajwvr=6&id=4141730615319112&page=4'
content = requests.get(url, cookies=cookies).text
assert len(comment.get_comment_list(content, '4141730615319112')) > 0
time.sleep(REQUEST_INTERNAL)
