def test_weibodata_oper(self):
db_session.execute("insert into {} ({}.weibo_id) values ('".format(
weibo_data.name, weibo_data.name) + FAKE_ID + "')")
assert WbDataOper.get_wb_by_mid(FAKE_ID) is not None
assert len(WbDataOper.get_weibo_comment_not_crawled()) == 1
assert len(WbDataOper.get_weibo_repost_not_crawled()) == 1
WbDataOper.set_weibo_comment_crawled(FAKE_ID)
WbDataOper.set_weibo_repost_crawled(FAKE_ID)
assert len(WbDataOper.get_weibo_comment_not_crawled()) == 0
assert len(WbDataOper.get_weibo_repost_not_crawled()) == 0
def test_weibodata_oper(self):
db_session.execute("insert into {} ({}.weibo_id) values ('".format(weibo_data.name, weibo_data.name)
+ FAKE_ID + "')")
assert WbDataOper.get_wb_by_mid(FAKE_ID) is not None
assert len(WbDataOper.get_weibo_comment_not_crawled()) == 1
assert len(WbDataOper.get_weibo_repost_not_crawled()) == 1
WbDataOper.set_weibo_comment_crawled(FAKE_ID)
WbDataOper.set_weibo_repost_crawled(FAKE_ID)
assert len(WbDataOper.get_weibo_comment_not_crawled()) == 0
assert len(WbDataOper.get_weibo_repost_not_crawled()) == 0
def execute_comment_task():
# 只解析了根评论，而未对根评论下的评论进行抓取，如果有需要的同学，可以适当做修改
weibo_datas = WbDataOper.get_weibo_comment_not_crawled()
for weibo_data in weibo_datas:
app.send_task('tasks.comment.crawl_comment_page',
args=(weibo_data.weibo_id, ),
queue='comment_crawler',
routing_key='comment_info')
def execute_comment_task():
weibo_datas = WbDataOper.get_weibo_comment_not_crawled(db_session)
crawl_comment_page(4253637545362266)
def execute_comment_task():
# 只解析了根评论，而未对根评论下的评论进行抓取，如果有需要的同学，可以适当做修改
weibo_datas = WbDataOper.get_weibo_comment_not_crawled()
for weibo_data in weibo_datas:
app.send_task('tasks.comment.crawl_comment_page', args=(weibo_data.weibo_id,), queue='comment_crawler',
routing_key='comment_info')