def CheckComment(client,dbc,weibo_id,last_comment_id=0):
now=time.time()
count=0
total_number=0
for page in xrange(1,100):
client.AddTask({'function':'comments__show','params':
{'id':str(weibo_id),'since_id':str(last_comment_id),'count':200,'trim_user':1,'page':page}})
headers,body=client.WaitResult()
weibores=json.loads(body)
comments=weibores.get('comments')
total_number=weibores.get('total_number',0)
if comments is None or len(comments)==0:
dbc.execute("update weibo_commentlast set checktime=? where weibo_id=?",(now,weibo_id))
print "%d comments for weibo %d"%(count,weibo_id)
return
last_one_comment=comments[0]
dbc.execute("replace into weibo_commentlast(weibo_id,last_comment_id,checktime) values(?,?,?)",(weibo_id,last_one_comment['id'],now))
for onec in comments:
reply_comment_id=0
if 'reply_comment' in onec:
reply_comment_id=onec['reply_comment']['id']
text=STTrans.getInstanse().TransT2S(onec['text'])
dbc.execute("replace into weibo_comment(weibo_id,comment_weibo_id,uid,reply_id,word) values(?,?,?,?,?)",(onec['id'],weibo_id,onec['uid'],reply_comment_id,text))
count+=len(comments)
if count>=total_number:
dbc.execute("update weibo_commentlast set checktime=? where weibo_id=?",(now,weibo_id))
print "%d comments for weibo %d (page end)"%(count,weibo_id)
return
def CheckComment(client, dbc, weibo_id, last_comment_id=0):
now = time.time()
count = 0
total_number = 0
for page in xrange(1, 100):
client.AddTask({
'function': 'comments__show',
'params': {
'id': str(weibo_id),
'since_id': str(last_comment_id),
'count': 200,
'trim_user': 1,
'page': page
}
})
headers, body = client.WaitResult()
weibores = json.loads(body)
comments = weibores.get('comments')
total_number = weibores.get('total_number', 0)
if comments is None or len(comments) == 0:
dbc.execute(
"update weibo_commentlast set checktime=? where weibo_id=?",
(now, weibo_id))
print "%d comments for weibo %d" % (count, weibo_id)
return
last_one_comment = comments[0]
dbc.execute(
"replace into weibo_commentlast(weibo_id,last_comment_id,checktime) values(?,?,?)",
(weibo_id, last_one_comment['id'], now))
for onec in comments:
reply_comment_id = 0
if 'reply_comment' in onec:
reply_comment_id = onec['reply_comment']['id']
text = STTrans.getInstanse().TransT2S(onec['text'])
dbc.execute(
"replace into weibo_comment(weibo_id,comment_weibo_id,uid,reply_id,word) values(?,?,?,?,?)",
(onec['id'], weibo_id, onec['uid'], reply_comment_id, text))
count += len(comments)
if count >= total_number:
dbc.execute(
"update weibo_commentlast set checktime=? where weibo_id=?",
(now, weibo_id))
print "%d comments for weibo %d (page end)" % (count, weibo_id)
return
print traceback.format_exc()
return
if len(all_time_line_statuses)>0:
last_one = all_time_line_statuses[0]
dbc=db.cursor()
dbc.execute("replace into weibo_lastweibo(user_id,last_weibo_id) values(?,?)",(client.user_id,last_one['id']))
db.commit()
dbc=db.cursor()
count=0
for one in all_time_line_statuses:
if 'uid' not in one or one['uid']==client.user_id:
continue
count+=1
text=STTrans.getInstanse().TransT2S(one['text'])
dbc.execute("insert or ignore into weibo_text(weibo_id,uid,word) values(?,?,?)",(one['id'],one['uid'],text))
dbc.execute("insert or ignore into weibo_commentlast(weibo_id,last_comment_id) values(?,?)",(one['id'],0))
db.commit()
db.close()
print 'read new weibo :%d'%len(all_time_line_statuses)
fetch_time+=1
def CheckComment(client,dbc,weibo_id,last_comment_id=0):
now=time.time()
count=0
total_number=0
for page in xrange(1,100):
client.AddTask({'function':'comments__show','params':
if len(all_time_line_statuses) > 0:
last_one = all_time_line_statuses[0]
dbc = db.cursor()
dbc.execute(
"replace into weibo_lastweibo(user_id,last_weibo_id) values(?,?)",
(client.user_id, last_one['id']))
db.commit()
dbc = db.cursor()
count = 0
for one in all_time_line_statuses:
if 'uid' not in one or one['uid'] == client.user_id:
continue
count += 1
text = STTrans.getInstanse().TransT2S(one['text'])
dbc.execute(
"insert or ignore into weibo_text(weibo_id,uid,word) values(?,?,?)",
(one['id'], one['uid'], text))
dbc.execute(
"insert or ignore into weibo_commentlast(weibo_id,last_comment_id) values(?,?)",
(one['id'], 0))
db.commit()
db.close()
print 'read new weibo :%d' % len(all_time_line_statuses)
fetch_time += 1
