def main(): initUid = const.initUid access_token = const.accessToken if queue.count()==0: queue.enqueue(initUid) while True: print '*******check name queue********' print 'step1: dequeue name' name = nameQueue.dequeue() if name is not None: print 'step2: get user by name : %s' % name try: user = client.getUserByName(name, access_token) if user is not None: print 'step3 save user : %s' % str(user) userStore.saveUser(user) print 'step4 put queue front, uid:%d' % user.uid queue.putFront(user.uid) except ApiException, e: print e if e.status == 403: print "reload config" const.loadConfig() access_token = const.accessToken nameQueue.enqueue(name) time.sleep(60) continue print '*******check uid queue********' print 'step1: dequeue' uid = queue.dequeue() if uid is None: break try: print 'step2: dump friends, uid=%d' % uid friends = client.dumpFriends(uid, access_token) except ApiException, e: print e if e.status == 403: print "reload config" const.loadConfig() access_token = const.accessToken queue.putFront(uid) time.sleep(60) continue
def main(): initUid = const.initUid access_token = const.accessToken if queue.count() == 0: queue.enqueue(initUid) while True: print '*******check name queue********' print 'step1: dequeue name' name = nameQueue.dequeue() if name is not None: print 'step2: get user by name : %s' % name try: user = client.getUserByName(name, access_token) if user is not None: print 'step3 save user : %s' % str(user) userStore.saveUser(user) print 'step4 put queue front, uid:%d' % user.uid queue.putFront(user.uid) except ApiException, e: print e if e.status == 403: print "reload config" const.loadConfig() access_token = const.accessToken nameQueue.enqueue(name) time.sleep(60) continue print '*******check uid queue********' print 'step1: dequeue' uid = queue.dequeue() if uid is None: break try: print 'step2: dump friends, uid=%d' % uid friends = client.dumpFriends(uid, access_token) except ApiException, e: print e if e.status == 403: print "reload config" const.loadConfig() access_token = const.accessToken queue.putFront(uid) time.sleep(60) continue
def runFlterEmpty(uids): global access_token print 'step1: get empty uids' for uid in uids: try: print 'step2: dump friends, uid=%d' % uid friends = client.dumpFriends(uid, access_token) except ApiException, e: print e if e.status == 403: print "reload config" const.loadConfig() access_token = const.accessToken time.sleep(60) continue friendIds = User.extractIds(friends) if len(friendIds)==0: print 'empty friends, uid=%d' % uid print friends friendsStore.delEmpty(uid) continue print 'step2: del empty uid' friendsStore.delEmpty(uid) print 'step3: save friend ids' friendsStore.saveFriends(uid, friendIds) print 'step4: save every friend in user store' userStore.saveUsers(friends) print 'step5: get enqueue list' existList = friendsStore.existPipe(friendIds) enqueueList = [] for i in range(len(friendIds)): if existList[i] == False: enqueueList.append(friendIds[i]) print 'step6: enqueue' queue.enqueuePipe(enqueueList) print 'friends.keyCount=%d' % friendsStore.keyCount() print '\n\n--------------------------------\n\n'
def runFlterEmpty(uids): global access_token print 'step1: get empty uids' for uid in uids: try: print 'step2: dump friends, uid=%d' % uid friends = client.dumpFriends(uid, access_token) except ApiException, e: print e if e.status == 403: print "reload config" const.loadConfig() access_token = const.accessToken time.sleep(60) continue friendIds = User.extractIds(friends) if len(friendIds) == 0: print 'empty friends, uid=%d' % uid print friends friendsStore.delEmpty(uid) continue print 'step2: del empty uid' friendsStore.delEmpty(uid) print 'step3: save friend ids' friendsStore.saveFriends(uid, friendIds) print 'step4: save every friend in user store' userStore.saveUsers(friends) print 'step5: get enqueue list' existList = friendsStore.existPipe(friendIds) enqueueList = [] for i in range(len(friendIds)): if existList[i] == False: enqueueList.append(friendIds[i]) print 'step6: enqueue' queue.enqueuePipe(enqueueList) print 'friends.keyCount=%d' % friendsStore.keyCount() print '\n\n--------------------------------\n\n'
else: queue.enqueue(uid) print teacher if teacher is not None: print 'step 3. check if teacher follows crawled' if not flagSet.exists('teacher:crawled', uid): print 'step 4. crawl follows' follows = client.getFollowTeachers(uid, access_token) flagSet.set('teacher:crawled', uid) for u in follows: print 'craw teacher', u print 'step 5. save teacher and enqueue teacher.uid' teacherStore.saveTeacher(u) teacherSort.add(u) queue.enqueue(u.uid) else: break except ApiException, e: print e if e.status == 403: print "reload config" const.loadConfig() access_token = const.accessToken if e.status != 400: queue.putFront(uid) time.sleep(60) continue if __name__ == '__main__': main()
queue.enqueue(uid) print teacher if teacher is not None: print 'step 3. check if teacher follows crawled' if not flagSet.exists('teacher:crawled', uid): print 'step 4. crawl follows' follows = client.getFollowTeachers(uid, access_token) flagSet.set('teacher:crawled', uid) for u in follows: print 'craw teacher', u print 'step 5. save teacher and enqueue teacher.uid' teacherStore.saveTeacher(u) teacherSort.add(u) queue.enqueue(u.uid) else: break except ApiException, e: print e if e.status == 403: print "reload config" const.loadConfig() access_token = const.accessToken if e.status != 400: queue.putFront(uid) time.sleep(60) continue if __name__ == '__main__': main()