def get_self_weibo_relation(selfUid): req = urllib2.Request(url='http://weibo.com/'+selfUid+'/myfollow',) result = urllib2.urlopen(req) text = result.read().decode('utf-8') # text = result.read() # 为了拿到自己的 昵称 总关注数目 总粉丝数 match = re.compile(u'class="gn_name" target="_top" title="[\s\S]*?"') searchResult = re.search(match, text) selfNick = searchResult.group(0)[37:-1].encode('utf-8') match = re.compile(u'全部关注\(\d+\)') searchResult = re.search(match, text) follows = searchResult.group(0)[5:-1] match = re.compile(u'粉丝\(\d+\)') searchResult = re.search(match, text) fans = searchResult.group(0)[3:-1] # 把自己加到数据库中 db.add_user(selfUid, selfNick, follows, fans) match = re.compile(u'uid=\d+') rawlv2 = re.findall(match, text) uidList = {}.fromkeys(rawlv2).keys() currentPageNum = 1 while len(uidList) < int(follows): print currentPageNum, len(uidList) match = re.compile(u'下一页') rawlv2 = re.findall(match, text) result = {}.fromkeys(rawlv2).keys() if len(result) > 0: currentPageNum += 1 req = urllib2.Request(url='http://weibo.com/'+selfUid+'/myfollow?t=1&page='+str(currentPageNum),) result = urllib2.urlopen(req) text = result.read().decode('utf-8') match = re.compile(u'uid=\d+') rawlv2 = re.findall(match, text) uidList += {}.fromkeys(rawlv2).keys() else: break print 'len(uidList)=',len(uidList) uidList = get_real_uid_list(uidList) uidList = list(set(uidList)) # remove duplicate element if selfUid in uidList: uidList.remove(selfUid) print uidList if len(uidList) > 0: for uid in uidList: db.add_relation(selfUid, uid) get_userinfo(uid) # 更新自己的 db_follows db_follows = db.count_db_follows(selfUid) db.update_user_db_follows(selfUid, db_follows)
def get_myrelation(): db.add_queue(UID) fans = [] for fan in weibo.get_myfans(UID): fans.append(fan['uid']) db.add_user(fan['uid'], fan['nickname'], fan['sex'], fan['address'], fan['info'], fan['face']) db.add_relation(UID, fan['uid']) follow = [] for fo in weibo.get_myfollow(UID): follow.append(fo['uid']) db.add_user(fo['uid'], fo['nickname'], fo['sex'], fo['address'], fo['info'], fo['face']) db.add_relation(fo['uid'], UID) for friend in get_friends(fans, follow): db.add_queue(friend) db.finish_queue(UID)
def get_relation(uid): fans = [] for fan in weibo.get_hisfans(uid): fans.append(fan['uid']) print('User[%s] fan[%s]'%(uid, fan['uid'])) db.add_user(fan['uid'], fan['nickname'], fan['sex'], fan['address'], fan['info'], fan['face']) db.add_relation(uid, fan['uid']) follow = [] for fo in weibo.get_hisfollow(uid): follow.append(fo['uid']) print('User[%s] follow[%s]'%(uid, fo['uid'])) db.add_user(fo['uid'], fo['nickname'], fo['sex'], fo['address'], fo['info'], fo['face']) db.add_relation(fo['uid'], uid) for friend in get_friends(fans, follow): db.add_queue(friend) print('User[%s] fans[%s] follow[%s]'%(uid, len(fans), len(follow))) db.finish_queue(uid)
def get_weibo_relation(uid): if not db.is_user_exist(uid): get_userinfo(uid) user = db.query_user(uid) if user: try: totalFollowsNum = user[2] req = urllib2.Request(url='http://weibo.com/'+uid+'/follow') result = urllib2.urlopen(req) text = result.read().decode('utf-8') match = re.compile(u'uid=\d+') rawlv2 = re.findall(match, text) uidList = {}.fromkeys(rawlv2).keys() currentPageNum = 1 print currentPageNum, 'uidList=', len(uidList) while len(uidList) < totalFollowsNum: match = re.compile(u'下一页') rawlv2 = re.findall(match, text) result = {}.fromkeys(rawlv2).keys() if len(result) > 0: req = urllib2.Request(url='http://weibo.com/'+uid+'/follow?page='+str(currentPageNum)) result = urllib2.urlopen(req) text = result.read().decode('utf-8') match = re.compile(u'uid=\d+') rawlv2 = re.findall(match, text) uidList += {}.fromkeys(rawlv2).keys() uidList.remove('uid='+uid) currentPageNum += 1 print currentPageNum, 'uidList=', len(uidList) if currentPageNum > 11: break print 'len(uidList)=',len(uidList) uidList = get_real_uid_list(uidList) if len(uidList) > 0: for u in uidList: db.add_relation(uid, u) get_userinfo(u) # 更新自己的 db_follows db_follows = db.count_db_follows(uid) db.update_user_db_follows(uid, db_follows) return 1 except Exception, e: print '>>>[Error: get_weibo_relation]', uid, e
def get_relation(uid): fans = [] for fan in weibo.get_hisfans(uid): fans.append(fan['uid']) print('User[%s] fan[%s]' % (uid, fan['uid'])) db.add_user(fan['uid'], fan['nickname'], fan['sex'], fan['address'], fan['info'], fan['face']) db.add_relation(uid, fan['uid']) follow = [] for fo in weibo.get_hisfollow(uid): follow.append(fo['uid']) print('User[%s] follow[%s]' % (uid, fo['uid'])) db.add_user(fo['uid'], fo['nickname'], fo['sex'], fo['address'], fo['info'], fo['face']) db.add_relation(fo['uid'], uid) for friend in get_friends(fans, follow): db.add_queue(friend) print('User[%s] fans[%s] follow[%s]' % (uid, len(fans), len(follow))) db.finish_queue(uid)