def crawl_repost_by_page(mid, page_num):
cur_url = BASE_URL.format(mid, page_num)
html = get_page(cur_url, auth_level=1, is_ajax=True)
repost_datas = repost.get_repost_list(html, mid)
if page_num == 1:
WbDataOper.set_weibo_repost_crawled(mid)
return html, repost_datas
def crawl_repost_by_page(mid, page_num):
cur_url = BASE_URL.format(mid, page_num)
html = get_page(cur_url, auth_level=1, is_ajax=True)
repost_datas = repost.get_repost_list(html, mid)
if page_num == 1:
WbDataOper.set_weibo_repost_crawled(mid)
return html, repost_datas
def test_weibodata_oper(self):
db_session.execute("insert into {} ({}.weibo_id) values ('".format(
weibo_data.name, weibo_data.name) + FAKE_ID + "')")
assert WbDataOper.get_wb_by_mid(FAKE_ID) is not None
assert len(WbDataOper.get_weibo_comment_not_crawled()) == 1
assert len(WbDataOper.get_weibo_repost_not_crawled()) == 1
WbDataOper.set_weibo_comment_crawled(FAKE_ID)
WbDataOper.set_weibo_repost_crawled(FAKE_ID)
assert len(WbDataOper.get_weibo_comment_not_crawled()) == 0
assert len(WbDataOper.get_weibo_repost_not_crawled()) == 0
def test_weibodata_oper(self):
db_session.execute("insert into {} ({}.weibo_id) values ('".format(weibo_data.name, weibo_data.name)
+ FAKE_ID + "')")
assert WbDataOper.get_wb_by_mid(FAKE_ID) is not None
assert len(WbDataOper.get_weibo_comment_not_crawled()) == 1
assert len(WbDataOper.get_weibo_repost_not_crawled()) == 1
WbDataOper.set_weibo_comment_crawled(FAKE_ID)
WbDataOper.set_weibo_repost_crawled(FAKE_ID)
assert len(WbDataOper.get_weibo_comment_not_crawled()) == 0
assert len(WbDataOper.get_weibo_repost_not_crawled()) == 0