def CheckComment(client,dbc,weibo_id,last_comment_id=0): now=time.time() count=0 total_number=0 for page in xrange(1,100): client.AddTask({'function':'comments__show','params': {'id':str(weibo_id),'since_id':str(last_comment_id),'count':200,'trim_user':1,'page':page}}) headers,body=client.WaitResult() weibores=json.loads(body) comments=weibores.get('comments') total_number=weibores.get('total_number',0) if comments is None or len(comments)==0: dbc.execute("update weibo_commentlast set checktime=? where weibo_id=?",(now,weibo_id)) print "%d comments for weibo %d"%(count,weibo_id) return last_one_comment=comments[0] dbc.execute("replace into weibo_commentlast(weibo_id,last_comment_id,checktime) values(?,?,?)",(weibo_id,last_one_comment['id'],now)) for onec in comments: reply_comment_id=0 if 'reply_comment' in onec: reply_comment_id=onec['reply_comment']['id'] text=STTrans.getInstanse().TransT2S(onec['text']) dbc.execute("replace into weibo_comment(weibo_id,comment_weibo_id,uid,reply_id,word) values(?,?,?,?,?)",(onec['id'],weibo_id,onec['uid'],reply_comment_id,text)) count+=len(comments) if count>=total_number: dbc.execute("update weibo_commentlast set checktime=? where weibo_id=?",(now,weibo_id)) print "%d comments for weibo %d (page end)"%(count,weibo_id) return
def CheckComment(client, dbc, weibo_id, last_comment_id=0): now = time.time() count = 0 total_number = 0 for page in xrange(1, 100): client.AddTask({ 'function': 'comments__show', 'params': { 'id': str(weibo_id), 'since_id': str(last_comment_id), 'count': 200, 'trim_user': 1, 'page': page } }) headers, body = client.WaitResult() weibores = json.loads(body) comments = weibores.get('comments') total_number = weibores.get('total_number', 0) if comments is None or len(comments) == 0: dbc.execute( "update weibo_commentlast set checktime=? where weibo_id=?", (now, weibo_id)) print "%d comments for weibo %d" % (count, weibo_id) return last_one_comment = comments[0] dbc.execute( "replace into weibo_commentlast(weibo_id,last_comment_id,checktime) values(?,?,?)", (weibo_id, last_one_comment['id'], now)) for onec in comments: reply_comment_id = 0 if 'reply_comment' in onec: reply_comment_id = onec['reply_comment']['id'] text = STTrans.getInstanse().TransT2S(onec['text']) dbc.execute( "replace into weibo_comment(weibo_id,comment_weibo_id,uid,reply_id,word) values(?,?,?,?,?)", (onec['id'], weibo_id, onec['uid'], reply_comment_id, text)) count += len(comments) if count >= total_number: dbc.execute( "update weibo_commentlast set checktime=? where weibo_id=?", (now, weibo_id)) print "%d comments for weibo %d (page end)" % (count, weibo_id) return
print traceback.format_exc() return if len(all_time_line_statuses)>0: last_one = all_time_line_statuses[0] dbc=db.cursor() dbc.execute("replace into weibo_lastweibo(user_id,last_weibo_id) values(?,?)",(client.user_id,last_one['id'])) db.commit() dbc=db.cursor() count=0 for one in all_time_line_statuses: if 'uid' not in one or one['uid']==client.user_id: continue count+=1 text=STTrans.getInstanse().TransT2S(one['text']) dbc.execute("insert or ignore into weibo_text(weibo_id,uid,word) values(?,?,?)",(one['id'],one['uid'],text)) dbc.execute("insert or ignore into weibo_commentlast(weibo_id,last_comment_id) values(?,?)",(one['id'],0)) db.commit() db.close() print 'read new weibo :%d'%len(all_time_line_statuses) fetch_time+=1 def CheckComment(client,dbc,weibo_id,last_comment_id=0): now=time.time() count=0 total_number=0 for page in xrange(1,100): client.AddTask({'function':'comments__show','params':
if len(all_time_line_statuses) > 0: last_one = all_time_line_statuses[0] dbc = db.cursor() dbc.execute( "replace into weibo_lastweibo(user_id,last_weibo_id) values(?,?)", (client.user_id, last_one['id'])) db.commit() dbc = db.cursor() count = 0 for one in all_time_line_statuses: if 'uid' not in one or one['uid'] == client.user_id: continue count += 1 text = STTrans.getInstanse().TransT2S(one['text']) dbc.execute( "insert or ignore into weibo_text(weibo_id,uid,word) values(?,?,?)", (one['id'], one['uid'], text)) dbc.execute( "insert or ignore into weibo_commentlast(weibo_id,last_comment_id) values(?,?)", (one['id'], 0)) db.commit() db.close() print 'read new weibo :%d' % len(all_time_line_statuses) fetch_time += 1