def main():
initUid = const.initUid
access_token = const.accessToken
if queue.count()==0:
queue.enqueue(initUid)
while True:
print '*******check name queue********'
print 'step1: dequeue name'
name = nameQueue.dequeue()
if name is not None:
print 'step2: get user by name : %s' % name
try:
user = client.getUserByName(name, access_token)
if user is not None:
print 'step3 save user : %s' % str(user)
userStore.saveUser(user)
print 'step4 put queue front, uid:%d' % user.uid
queue.putFront(user.uid)
except ApiException, e:
print e
if e.status == 403:
print "reload config"
const.loadConfig()
access_token = const.accessToken
nameQueue.enqueue(name)
time.sleep(60)
continue
print '*******check uid queue********'
print 'step1: dequeue'
uid = queue.dequeue()
if uid is None:
break
try:
print 'step2: dump friends, uid=%d' % uid
friends = client.dumpFriends(uid, access_token)
except ApiException, e:
print e
if e.status == 403:
print "reload config"
const.loadConfig()
access_token = const.accessToken
queue.putFront(uid)
time.sleep(60)
continue
def main():
initUid = const.initUid
access_token = const.accessToken
if queue.count() == 0:
queue.enqueue(initUid)
while True:
print '*******check name queue********'
print 'step1: dequeue name'
name = nameQueue.dequeue()
if name is not None:
print 'step2: get user by name : %s' % name
try:
user = client.getUserByName(name, access_token)
if user is not None:
print 'step3 save user : %s' % str(user)
userStore.saveUser(user)
print 'step4 put queue front, uid:%d' % user.uid
queue.putFront(user.uid)
except ApiException, e:
print e
if e.status == 403:
print "reload config"
const.loadConfig()
access_token = const.accessToken
nameQueue.enqueue(name)
time.sleep(60)
continue
print '*******check uid queue********'
print 'step1: dequeue'
uid = queue.dequeue()
if uid is None:
break
try:
print 'step2: dump friends, uid=%d' % uid
friends = client.dumpFriends(uid, access_token)
except ApiException, e:
print e
if e.status == 403:
print "reload config"
const.loadConfig()
access_token = const.accessToken
queue.putFront(uid)
time.sleep(60)
continue
def runFlterEmpty(uids):
global access_token
print 'step1: get empty uids'
for uid in uids:
try:
print 'step2: dump friends, uid=%d' % uid
friends = client.dumpFriends(uid, access_token)
except ApiException, e:
print e
if e.status == 403:
print "reload config"
const.loadConfig()
access_token = const.accessToken
time.sleep(60)
continue
friendIds = User.extractIds(friends)
if len(friendIds)==0:
print 'empty friends, uid=%d' % uid
print friends
friendsStore.delEmpty(uid)
continue
print 'step2: del empty uid'
friendsStore.delEmpty(uid)
print 'step3: save friend ids'
friendsStore.saveFriends(uid, friendIds)
print 'step4: save every friend in user store'
userStore.saveUsers(friends)
print 'step5: get enqueue list'
existList = friendsStore.existPipe(friendIds)
enqueueList = []
for i in range(len(friendIds)):
if existList[i] == False:
enqueueList.append(friendIds[i])
print 'step6: enqueue'
queue.enqueuePipe(enqueueList)
print 'friends.keyCount=%d' % friendsStore.keyCount()
print '\n\n--------------------------------\n\n'
def runFlterEmpty(uids):
global access_token
print 'step1: get empty uids'
for uid in uids:
try:
print 'step2: dump friends, uid=%d' % uid
friends = client.dumpFriends(uid, access_token)
except ApiException, e:
print e
if e.status == 403:
print "reload config"
const.loadConfig()
access_token = const.accessToken
time.sleep(60)
continue
friendIds = User.extractIds(friends)
if len(friendIds) == 0:
print 'empty friends, uid=%d' % uid
print friends
friendsStore.delEmpty(uid)
continue
print 'step2: del empty uid'
friendsStore.delEmpty(uid)
print 'step3: save friend ids'
friendsStore.saveFriends(uid, friendIds)
print 'step4: save every friend in user store'
userStore.saveUsers(friends)
print 'step5: get enqueue list'
existList = friendsStore.existPipe(friendIds)
enqueueList = []
for i in range(len(friendIds)):
if existList[i] == False:
enqueueList.append(friendIds[i])
print 'step6: enqueue'
queue.enqueuePipe(enqueueList)
print 'friends.keyCount=%d' % friendsStore.keyCount()
print '\n\n--------------------------------\n\n'
else:
queue.enqueue(uid)
print teacher
if teacher is not None:
print 'step 3. check if teacher follows crawled'
if not flagSet.exists('teacher:crawled', uid):
print 'step 4. crawl follows'
follows = client.getFollowTeachers(uid, access_token)
flagSet.set('teacher:crawled', uid)
for u in follows:
print 'craw teacher', u
print 'step 5. save teacher and enqueue teacher.uid'
teacherStore.saveTeacher(u)
teacherSort.add(u)
queue.enqueue(u.uid)
else:
break
except ApiException, e:
print e
if e.status == 403:
print "reload config"
const.loadConfig()
access_token = const.accessToken
if e.status != 400:
queue.putFront(uid)
time.sleep(60)
continue
if __name__ == '__main__':
main()
queue.enqueue(uid)
print teacher
if teacher is not None:
print 'step 3. check if teacher follows crawled'
if not flagSet.exists('teacher:crawled', uid):
print 'step 4. crawl follows'
follows = client.getFollowTeachers(uid, access_token)
flagSet.set('teacher:crawled', uid)
for u in follows:
print 'craw teacher', u
print 'step 5. save teacher and enqueue teacher.uid'
teacherStore.saveTeacher(u)
teacherSort.add(u)
queue.enqueue(u.uid)
else:
break
except ApiException, e:
print e
if e.status == 403:
print "reload config"
const.loadConfig()
access_token = const.accessToken
if e.status != 400:
queue.putFront(uid)
time.sleep(60)
continue
if __name__ == '__main__':
main()
