def grab(url, threadID): print url user_grab = User(url) followers = user_grab.get_followers() for i, user_grab in enumerate(followers): user = Users() while True: try: if Users.objects(data_id = user_grab.get_data_id()).count(): user = Users.objects(data_id = user_grab.get_data_id()).first() break except Exception, e: logging.error("========error1") logging.error(e) time.sleep(300) user.user_id = user_grab.get_user_id() user.data_id = user_grab.get_data_id() user.followees_num = user_grab.get_followees_num() user.followers_num = user_grab.get_followers_num() try: user.asks_num = user_grab.get_asks_num() user.answers_num = user_grab.get_answers_num() user.collections_num = user_grab.get_collections_num() except Exception, e: logging.error("========error2") logging.error(e) logging.debug(user_grab.get_user_url())
def user_test(user_url): user = User(user_url) # 获取用户ID user_id = user.get_user_id() # 获取该用户的关注者人数 followers_num = user.get_followers_num() # 获取该用户关注的人数 followees_num = user.get_followees_num() # 获取该用户提问的个数 asks_num = user.get_asks_num() # 获取该用户回答的个数 answers_num = user.get_answers_num() # 获取该用户收藏夹个数 collections_num = user.get_collections_num() # 获取该用户获得的赞同数 agree_num = user.get_agree_num() # 获取该用户获得的感谢数 thanks_num = user.get_thanks_num() # 获取该用户关注的人 followees = user.get_followees() # 获取关注该用户的人 followers = user.get_followers() # 获取该用户提的问题 asks = user.get_asks() # 获取该用户回答的问题的答案 answers = user.get_answers() # 获取该用户的收藏夹 collections = user.get_collections() print user_id # 黄继新 print followers_num # 614840 print followees_num # 8408 print asks_num # 1323 print answers_num # 786 print collections_num # 44 print agree_num # 46387 print thanks_num # 11477 print followees # <generator object get_followee at 0x7ffcac3af050> # 代表所有该用户关注的人的生成器对象 print followers # <generator object get_follower at 0x7ffcac3af0f0> # 代表所有关注该用户的人的生成器对象 print asks # <generator object get_ask at 0x7ffcab9db780> # 代表该用户提的所有问题的生成器对象 print answers # <generator object get_answer at 0x7ffcab9db7d0> # 代表该用户回答的所有问题的答案的生成器对象 print collections
def user_test(user_url): user = User(user_url) # 获取用户ID user_id = user.get_user_id() # 获取该用户的关注者人数 followers_num = user.get_followers_num() # 获取该用户关注的人数 followees_num =user.get_followees_num() # 获取该用户提问的个数 asks_num = user.get_asks_num() # 获取该用户回答的个数 answers_num = user.get_answers_num() # 获取该用户收藏夹个数 collections_num = user.get_collections_num() # 获取该用户获得的赞同数 agree_num = user.get_agree_num() # 获取该用户获得的感谢数 thanks_num = user.get_thanks_num() # 获取该用户关注的人 followees = user.get_followees() # 获取关注该用户的人 followers = user.get_followers() # 获取该用户提的问题 asks = user.get_asks() # 获取该用户回答的问题的答案 answers = user.get_answers() # 获取该用户的收藏夹 collections = user.get_collections() print user_id # 黄继新 print followers_num # 614840 print followees_num # 8408 print asks_num # 1323 print answers_num # 786 print collections_num # 44 print agree_num # 46387 print thanks_num # 11477 print followees # <generator object get_followee at 0x7ffcac3af050> # 代表所有该用户关注的人的生成器对象 print followers # <generator object get_follower at 0x7ffcac3af0f0> # 代表所有关注该用户的人的生成器对象 print asks # <generator object get_ask at 0x7ffcab9db780> # 代表该用户提的所有问题的生成器对象 print answers # <generator object get_answer at 0x7ffcab9db7d0> # 代表该用户回答的所有问题的答案的生成器对象 print collections
user_collections = user.get_collections() for collection in user_collections: # 输出每一个收藏夹的名字 print collection.get_name() # 得到该收藏夹下的前十个回答 top_answers = collection.get_top_i_answers(10) # 把答案内容转成txt,markdown for answer in top_answers: answer.to_txt() answer.to_md() def main(): url = "http://www.zhihu.com/question/24269892" question_test(url) answer_url = "http://www.zhihu.com/question/24269892/answer/29960616" answer_test(answer_url) user_url = "http://www.zhihu.com/people/jixin" user_test(user_url) collection_url = "http://www.zhihu.com/collection/36750683" collection_test(collection_url) test() if __name__ == '__main__': user_url = "https://www.zhihu.com/people/BravoMaooo" user = User(user_url) followers = user.get_followers() for follower in followers: print(follower.get_user_id())
def user_test(user_url): user = User(user_url) # 获取用户ID user_id = user.get_user_id() # 获取用户性别 user_gender = user.get_gender() # 获取该用户的关注者人数 followers_num = user.get_followers_num() # 获取该用户关注的人数 followees_num = user.get_followees_num() # 获取该用户提问的个数 asks_num = user.get_asks_num() # 获取该用户回答的个数 answers_num = user.get_answers_num() # 获取该用户收藏夹个数 collections_num = user.get_collections_num() # 获取该用户获得的赞同数 agree_num = user.get_agree_num() # 获取该用户获得的感谢数 thanks_num = user.get_thanks_num() # 获取该用户的头像url head_img_url = user.get_head_img_url() # 获取该用户关注的人 followees = user.get_followees() # 获取关注该用户的人 followers = user.get_followers() # 获取该用户提的问题 asks = user.get_asks() # 获取该用户回答的问题的答案 answers = user.get_answers() # 获取该用户的收藏夹 collections = user.get_collections() print user_id # 黄继新 print user_gender #male print followers_num # 614840 print followees_num # 8408 print asks_num # 1323 print answers_num # 786 print collections_num # 44 print agree_num # 46387 print thanks_num # 11477 print head_img_url # https://pic2.zhimg.com/0626f4164009f291b26a79d96c6962c5_l.jpg print followees # <generator object get_followee at 0x7ffcac3af050> # 代表所有该用户关注的人的生成器对象 i = 0 for followee in followees: print followee.get_user_id() i = i + 1 if i == 41: break print followers # <generator object get_follower at 0x7ffcac3af0f0> # 代表所有关注该用户的人的生成器对象 i = 0 for follower in followers: print follower.get_user_id() i = i + 1 if i == 41: break print asks # <generator object get_ask at 0x7ffcab9db780> # 代表该用户提的所有问题的生成器对象 print answers # <generator object get_answer at 0x7ffcab9db7d0> # 代表该用户回答的所有问题的答案的生成器对象 print collections
def user_test(user_url): user = User(user_url) # 获取用户ID user_id = user.get_user_id() # 获取用户性别 user_gender = user.get_gender() # 获取该用户的关注者人数 followers_num = user.get_followers_num() # 获取该用户关注的人数 followees_num = user.get_followees_num() # 获取该用户提问的个数 asks_num = user.get_asks_num() # 获取该用户回答的个数 answers_num = user.get_answers_num() # 获取该用户收藏夹个数 collections_num = user.get_collections_num() # 获取该用户获得的赞同数 agree_num = user.get_agree_num() # 获取该用户获得的感谢数 thanks_num = user.get_thanks_num() # 获取该用户的头像url head_img_url = user.get_head_img_url() # 获取该用户关注的人 followees = user.get_followees() # 获取关注该用户的人 followers = user.get_followers() topics = user.get_topics() # 获取该用户提的问题 asks = user.get_asks() # 获取该用户回答的问题的答案 answers = user.get_answers() # 获取该用户的收藏夹 collections = user.get_collections() print user_id # 黄继新 print user_gender #male print followers_num # 614840 print followees_num # 8408 print asks_num # 1323 print answers_num # 786 print collections_num # 44 print agree_num # 46387 print thanks_num # 11477 print head_img_url # https://pic2.zhimg.com/0626f4164009f291b26a79d96c6962c5_l.jpg print followees # <generator object get_followee at 0x7ffcac3af050> # 代表所有该用户关注的人的生成器对象 i = 0 for followee in followees: print followee.get_user_id() i = i + 1 if i == 41: break print followers # <generator object get_follower at 0x7ffcac3af0f0> # 代表所有关注该用户的人的生成器对象 i = 0 for follower in followers: print follower.get_user_id() i = i + 1 if i == 41: break for topic in topics: print topic print asks # <generator object get_ask at 0x7ffcab9db780> # 代表该用户提的所有问题的生成器对象 print answers # <generator object get_answer at 0x7ffcab9db7d0> # 代表该用户回答的所有问题的答案的生成器对象 print collections
def user_spider(user_url): database_name = 'wjw_zhihu' table_name = 'user_info' # 设置数据库连接 conn=pymysql.connect(host='localhost',user='******',passwd='root',port=3306) cur=conn.cursor() # 选择数据库 conn.select_db(database_name) # 设置编码, 否则插入数据库乱码 cur.execute('set names utf8') # 设置Redis链接, 记录爬过的user_unique redis_conn = redis.Redis(host='127.0.0.1', port=6379, db=0) # 获取当前用户信息 user = User(user_url) user_unique = user.get_user_unique() if redis_conn.get(get_user_redis_key(user_unique)) == None: user_info = user.get_user_info() # print user_info; # sys.exit() # 将用户数据插入数据库 try: insert_sql = prepare_insert_sql(table_name, user_info) res=cur.execute(insert_sql) conn.commit() # commit之后才能真正提交到数据库 redis_conn.set(get_user_redis_key(user_unique), 1) #设置redis缓存, 防止重爬 print(user_info['user_unique'] + ' ------ ' + str(res)) except Exception as e: # 打印日志, 记录异常信息 exceptMsg = str(e) print(exceptMsg) # 获取该用户关注的人 followees = user.get_followees() # 获取关注该用户的人 followers = user.get_followers() # 统计该用户关注的人 # i = 0 # for followee in followees: # print followee.user_url # print followee.get_user_id() # i = i + 1 # if i == 41: # break # print followers # <generator object get_follower at 0x7ffcac3af0f0> # 代表所有关注该用户的人的生成器对象 i = 0 for follower in followers: i = i + 1 if i % 10 == 0: redis_conn.save() # 将数据写回磁盘。保存时阻塞 time.sleep(0.3) follower_user_unique = follower.get_user_unique() if redis_conn.get(get_user_redis_key(follower_user_unique)) == None: try: follower_info = follower.get_user_info() follower_insert_sql = prepare_insert_sql(table_name, follower_info) res=cur.execute(follower_insert_sql) conn.commit() redis_conn.set(get_user_redis_key(follower_user_unique), 1) #设置redis缓存, 防止重爬 print(follower_info['user_unique'] + ' ------ ' + str(res)) except Exception as e: # 打印日志, 记录异常信息 exceptMsg = str(e) print(exceptMsg)
str=follower.get_work() print(str) if allWork.has_key(str): allWork[str]+=1 else: allWork[str]=1 print json.dumps(allWork, encoding="UTF-8", ensure_ascii=False) #to delete bias if 'unknown' in allWork: del allWork['unknown'] top5Cities = dict(sorted(allWork.iteritems(), key=operator.itemgetter(1), reverse=True)[:5]) return top5Cities def getTop5Relations(followers): allFollowers={} for follower in followers: allFollowers[follower.get_user_id()]=follower.get_vote_thank_relation() print(allFollowers) superFriends = dict(sorted(allFollowers.iteritems(), key=operator.itemgetter(1), reverse=True)[:5]) return superFriends if __name__ == '__main__': #an example to get your friends' city location user_url = "https://www.zhihu.com/people/BravoMaooo" user = User(user_url) followers = user.get_followers() dics=getTop5Works(followers) print(dics) v.plotPie4Top5(dics)
def user_test(user_url): user = User(user_url) # 获取用户ID user_id = user.get_user_id() # 获取该用户的关注者人数 followers_num = user.get_followers_num() # 获取该用户关注的人数 followees_num = user.get_followees_num() # 获取该用户提问的个数 asks_num = user.get_asks_num() # 获取该用户回答的个数 answers_num = user.get_answers_num() # 获取该用户收藏夹个数 collections_num = user.get_collections_num() # 获取该用户获得的赞同数 agree_num = user.get_agree_num() # 获取该用户获得的感谢数 thanks_num = user.get_thanks_num() # 获取该用户关注的人 followees = user.get_followees() # 获取关注该用户的人 followers = user.get_followers() # 获取该用户提的问题 asks = user.get_asks() # 获取该用户回答的问题的答案 answers = user.get_answers() # 获取该用户2014年回答的问题的答案 answers_in_2014 = user.get_answers(begin_date='2014-01-01', end_date='2014-12-31') # 获取该用户的收藏夹 collections = user.get_collections() print user_id # 黄继新 print followers_num # 614840 print followees_num # 8408 print asks_num # 1323 print answers_num # 786 print collections_num # 44 print agree_num # 46387 print thanks_num # 11477 print followees # <generator object get_followee at 0x7ffcac3af050> # 代表所有该用户关注的人的生成器对象 i = 0 for followee in followees: print followee.get_user_id() i = i + 1 if i == 41: break print followers # <generator object get_follower at 0x7ffcac3af0f0> # 代表所有关注该用户的人的生成器对象 i = 0 for follower in followers: print follower.get_user_id() i = i + 1 if i == 41: break print asks # <generator object get_ask at 0x7ffcab9db780> # 代表该用户提的所有问题的生成器对象 print answers # <generator object get_answer at 0x7ffcab9db7d0> # 代表该用户回答的所有问题的答案的生成器对象 print answers_in_2014 # 代表该用户2014年回答的所有问题的答案的生成器对象 print collections
def main(): client = pymongo.MongoClient("localhost", 27017) db = client.zhihu_user urllist = db.urllist userlist = db.userlist origin_users = ["https://www.zhihu.com/people/jixin", "https://www.zhihu.com/people/zhang-jia-wei", "https://www.zhihu.com/people/zhu-xuan-86", "https://www.zhihu.com/people/kaifulee", "https://www.zhihu.com/people/e-miao-de-nai-ba"] urls = urllist.distinct("user_url") for u in origin_users: if u in urls: pass else: urllist.insert({"user_url": u, "jlzt": "1"}) while 1: item = urllist.find_one({'jlzt':'1'}) if item == None: print u'已全部处理完成' break else: user_url = item["user_url"] starttime = datetime.datetime.now() urllist.update({"user_url":user_url},{"$set":{"jlzt":"3"}}) try: user = User(user_url) zhihu_id = user.get_data_id() # 用户唯一id zhihu_name = user.get_user_id() # 用户名 followees_num = user.get_followees_num() # 用户关注人数 followers_num = user.get_followers_num() # 用户关注者人数 gender = user.get_gender() # 性别 # 提问数 asks_num = user.get_asks_num() # 获取该用户回答的个数 answers_num = user.get_answers_num() # 获取该用户收藏夹个数 collections_num = user.get_collections_num() # 获取该用户获得的赞同数 agree_num = user.get_agree_num() # 获取该用户获得的感谢数 thanks_num = user.get_thanks_num() # 获取该用户的头像url head_img_url = user.get_head_img_url() # 关注的话题数 topics_num = user.get_topics_num() # 获取该用户关注的人 followees = user.get_followees() # 获取关注该用户的人 followers = user.get_followers() # 获取提出的问题 questions = user.get_asks() # 获取回答的问题 answers = user.get_answers() # 获取话题 topics = user.get_topics() print "start process " + zhihu_name + ";\n" #print zhihu_id # 黄继新 #print zhihu_name # 614840 #print followees_num # 8408 #print followers_num # 1323 #print gender # 786 #print asks_num # 44 #print answers_num # 46387 #print collections_num # 11477 #print agree_num #print thanks_num #print head_img_url #print topics_num followee_l = [] follower_l = [] questions_l = [] answers_l = [] topics_l = [] print u'开始处理关注的人' for followee in followees: followee_l.append(followee.user_url.split('/')[4]) if len(followee_l) % 100 == 0: print zhihu_name + "'s NO." + str(len(followee_l)) + " followee is being processed. please wait..." time.sleep(0.05) followee_list = ','.join(followee_l) print u'添加完成' print u'开始添加关注者至处理队列' for follower in followers: follower_l.append(follower.user_url.split('/')[4]) urls = urllist.distinct("user_url") if follower.user_url in urls: pass # print "follower_url:'"+follower.user_url+"' passed" else: urllist.insert({"user_url": follower.user_url, "jlzt": "1"}) #print "follower_url:'" +follower.user_url + "' added" if len(follower_l) % 100 == 0: print zhihu_name + "'s NO." + str(len(follower_l)) + " follower is being processed. please wait..." time.sleep(0.05) print u'添加完成' for q in questions: questions_l.append("url=" + q.url + "|title=" + q.get_title()) time.sleep(0.01) for a in answers: answers_l.append(a.answer_url) time.sleep(0.01) for t in topics: topics_l.append(t) time.sleep(0.01) user_data = {"zhihu_id":zhihu_id, "zhihu_name":zhihu_name, "followees_num":followees_num, "followers_num":followers_num, "followees":followee_l, "followers":follower_l, "questions":questions_l, "gender":gender, "asks_num":asks_num, "answers_num":answers_num, "ansers":answers_l, "collections_num":collections_num, "agree_num":agree_num, "thanks_num":thanks_num, "topics_num":topics_num, "topics":topics_l, "head_img_url":head_img_url } print "user_data prepared:" urls = userlist.distinct("user_url") if user_url in urls: pass else: userlist.insert(user_data) print "user_data inserted: \n" urllist.update({"user_url":user_url},{"$set":{"jlzt":"2"}}) endtime = datetime.datetime.now() interval=(endtime - starttime).seconds print zhihu_name + "finnished. spent " + str(interval) + "seconds." except: traceback.print_exc() time.sleep(10) urllist.update({"user_url":user_url},{"$set":{"jlzt":"1"}}) continue print "处理完毕"