def set_config(my_app):
ensure_dir(CONFIG_PATH)
try:
with open(CONFIG_PATH, 'r') as f:
data = f.readline()
if data:
try:
config = json.loads(data)
except ValueError:
return
except FileNotFoundError: # first run or delete config file
with open(CONFIG_PATH, 'w'):
pass
if data:
if config["is_sign"] and config["is_sign"] is not str or list:
try:
with open(COOKIES_PATH, 'r') as f:
FeedsList.client = ZhihuClient(COOKIES_PATH)
Noticer.client = ZhihuClient(COOKIES_PATH)
except FileNotFoundError: # first run or delete config file
with open(CONFIG_PATH, 'w'):
pass
else:
FeedsList.client = ZhihuClient()
Noticer.client = ZhihuClient()
if config["proxy"]:
FeedsList.client.set_proxy(config["proxy"])
Noticer.client.set_proxy(config["proxy"])
def init_db(cookies):
if cookies is not None:
c.check_type(cookies, 'cookies', str)
# login in with cookies file
if cookies:
author = ZhihuClient(cookies=cookies).me()
# login in terminal
else:
client = ZhihuClient()
try:
cookies = client.login_in_terminal()
except KeyboardInterrupt:
print()
cookies = ''
if not cookies:
L.error(s.log_login_failed)
L.info(s.exit)
exit(0)
author = client.me()
try:
conn = db.create_db(author)
db.create_table(conn)
db.dump_init_data_to_db(conn, author)
db.close_db(conn)
print(s.success)
except FileExistsError as e:
L.error(s.file_exist.format(e.filename))
print(s.failed)
def init_db(cookies):
if cookies is not None:
c.check_type(cookies, 'cookies', str)
# login in with cookies file
if cookies:
author = ZhihuClient(cookies=cookies).me()
# login in terminal
else:
client = ZhihuClient()
try:
cookies = client.login_in_terminal()
except KeyboardInterrupt:
print()
cookies = ''
if not cookies:
L.error(s.log_login_failed)
L.info(s.exit)
exit(0)
author = client.me()
try:
conn = db.create_db(author)
db.create_table(conn)
db.dump_init_data_to_db(conn, author)
db.close_db(conn)
print(s.success)
except FileExistsError as e:
L.error(s.file_exist.format(e.filename))
print(s.failed)
def spider_login(self, cookie=None):
client = ZhihuClient()
if cookie == None:
cookie_file_name = ''
else:
cookie_file_name = cookie
if os.path.exists(cookie_file_name):
client.login_with_cookies(cookie_file_name)
else:
client.create_cookies(cookie_file_name)
return client
def set_proxy():
zc = ZhihuClient()
ensure_dir(PROXY_PATH)
try:
with open(PROXY_PATH, "r") as f:
proxy = f.read()
if proxy:
zc.set_proxy(proxy)
except FileNotFoundError: # first run or delete config file
if os.path.exists(os.path.dirname(PROXY_PATH)) is False:
os.makedirs(os.path.dirname(PROXY_PATH))
with open(PROXY_PATH, "w"):
pass
def sample_code():
client = ZhihuClient()
cookie_file_name = 'cookie.json'
people_main_page = 'http://www.zhihu.com/people/liu-shi-qi-5-21'
if os.path.exists(cookie_file_name):
client.login_with_cookies(cookie_file_name)
else:
client.create_cookies(cookie_file_name)
me = client.author(people_main_page)
print('id:', me.id)
print('name:', me.name)
print('motto:', me.motto)
print('photo:', me.photo_url)
print('followee number:', me.followee_num)
print('follower number:', me.follower_num)
for i in range(1000):
print(me.motto)
followees = []
followers = []
for man in me.followees:
print(man.id, '/', man.name)
followees.append(man.id)
for man in me.followers:
print(man.id, '/', man.name)
followers.append(man.id)
save = ZhihuUser(me.id, followees, followers)
save.save('save.json')
# ==============================
USER_URL = "https://www.zhihu.com/people/7sdream"
FOLLOWER_CHECK_MAX_NUM = 2000
ANSWER_CHECK_MAX_NUM = 20
# ==============================
def is_zero_user(author):
return (author.upvote_num + author.question_num + author.answer_num) <= 3
client = ZhihuClient('test.json')
user = client.author(USER_URL)
print("检查用户{user.name} at {time}".format(user=user,
time=datetime.datetime.now()))
if user.follower_num < FOLLOWER_CHECK_MAX_NUM:
FOLLOWER_CHECK_MAX_NUM = user.follower_num
print("正在检查前{FOLLOWER_CHECK_MAX_NUM}个关注者....".format(**locals()))
zeros = 0
for _, follower in zip(range(FOLLOWER_CHECK_MAX_NUM), user.followers):
if is_zero_user(follower):
zeros += 1
TEST_DIR = os.path.join(BASE_DIR, 'test')
print("Test dir: ", TEST_DIR)
if os.path.exists(TEST_DIR):
print("Cleaning it...", end='')
shutil.rmtree(TEST_DIR)
print("Done")
else:
print("Test dir not exist.")
os.chdir(BASE_DIR)
if os.path.isfile(Cookies_File):
print("Cookies file found.")
client = ZhihuClient(Cookies_File)
else:
print("Cookies file not exist, please login...")
client = ZhihuClient()
cookies_str = client.login_in_terminal()
with open(Cookies_File, 'w') as f:
f.write(cookies_str)
print("Making test dir...", end="")
os.mkdir(TEST_DIR)
print("Done", end="\n\n")
os.chdir(TEST_DIR)
print("===== test start =====")
from RedisQueue import RedisQueue
from zhihu import ZhihuClient
import datetime
import time
import random
import sys
from timeout import timeout
from termcolor import colored, cprint
MAX_SLEEP_TIME = 15
Cookies_File = './cookies/cookies%s.json' % sys.argv[1]
global client
client = ZhihuClient(Cookies_File)
print_err = lambda x: cprint(x, 'red')
def get_user_info(uname):
global client
if uname == '':
return None
url = 'http://www.zhihu.com/people/%s' % uname
print(time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + url)
all_info = []
try:
author = client.author(url)
# 获取用户url
all_info.append(uname)
# 获取用户名字
all_info.append(author.name)
from zhihu import ZhihuClient
Cookies_File = 'cookies.json'
client = ZhihuClient(Cookies_File)
dict={}
author = client.author('https://www.zhihu.com/people/zhang-jia-wei')
#author = client.author('https://www.zhihu.com/people/mo-ming-42-91')
print('--- Followers ---')
for follower in author.followers:
if(follower.education!='unknown'):
print(follower.name,follower.education)
with open('zhihu.txt','a') as f:
f.write(follower.name+": "+follower.education+"\n")
if(follower.education in dict):
dict[follower.education]+=1
else:
dict[follower.education]=1
'''print('--- Followees ---')
for followee in author.followees:
print(followee.name,followee.education)'''
print(dict)
with open('zhihu.txt','a') as f:
f.write(str(dict)+"\n")
TEST_DIR = os.path.join(BASE_DIR, 'test')
print("Test dir: ", TEST_DIR)
if os.path.exists(TEST_DIR):
print("Cleaning it...", end='')
shutil.rmtree(TEST_DIR)
print("Done")
else:
print("Test dir not exist.")
os.chdir(BASE_DIR)
if os.path.isfile(Cookies_File):
print("Cookies file found.")
client = ZhihuClient(Cookies_File)
else:
print("Cookies file not exist, please login...")
client = ZhihuClient()
cookies_str = client.login_in_terminal()
with open(Cookies_File, 'w') as f:
f.write(cookies_str)
print("Making test dir...", end="")
os.mkdir(TEST_DIR)
print("Done", end="\n\n")
os.chdir(TEST_DIR)
print("===== test start =====")
from zhihu import ZhihuClient
ZhihuClient().create_cookies('cookies.json',
need_captcha=False,
use_getpass=False)
from zhihu import ZhihuClient
client=ZhihuClient()
client.create_cookies("zhihucookie.json")
from zhihu import ZhihuClient
import zhihu
Cookies_File = 'cookies.json'
client = ZhihuClient(Cookies_File)
url = 'https://www.zhihu.com/people/yanyang1025'
author = client.author(url)
print('取用户粉丝数 %d' % author.follower_num)
print('用户得到赞同数 %d' % author.upvote_num)
for act in author.activities:
if act.type == zhihu.ActType.UPVOTE_ANSWER:
print('%s 在 %s 赞同了问题 %s 中 %s(motto: %s) 的回答, '
'此回答赞同数 %d' %
(author.name, act.time, act.answer.question.title,
act.answer.author.name, act.answer.author.motto,
act.answer.upvote_num))
#for question in author.followed_questions:
# print(question.title)
def run(self, database, msg, interval, log_file, max_old=10):
c.check_type(database, 'database', str)
L = logging.getLogger('qqqfome-backend')
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(message)s')
fh = logging.FileHandler(log_file)
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
sh = logging.StreamHandler()
sh.setLevel(logging.DEBUG)
sh.setFormatter(formatter)
L.setLevel(logging.DEBUG)
L.addHandler(fh)
L.addHandler(sh)
try:
L.info(s.log_connected_to_db.format(database))
conn = db.connect_db(database)
L.info(s.success)
except FileNotFoundError:
L.exception(s.log_file_not_exist.format(database))
L.info(s.exit)
return
# get cookies from database
cookies = db.get_cookies(conn)
if not cookies:
L.exception(s.log_no_cookies_in_database)
L.info(s.exit)
return
L.info(s.log_get_cookies_from_database)
L.debug(cookies)
try:
client = ZhihuClient(cookies)
L.info(s.log_build_zhihu_client)
except Exception as e:
L.exception(e)
return
while True:
L.info(s.log_start_a_pass)
i = 0
while i < 5:
try:
L.info(s.log_build_me)
me = client.me()
break
except Exception as e:
L.exception(e)
i += 1
else:
L.error(s.log_fail_to_build_me)
L.info(s.exit)
return
try:
follower_num = me.follower_num
except Exception as e:
L.exception(e)
L.info(s.log_get_follower_num_failed)
L.info(s.log_finish_a_pass)
time.sleep(interval)
continue
L.info(s.log_get_follower_num.format(follower_num))
db.log_to_db(conn, follower_num, s.log_start_a_pass)
continue_in_db = 0
new_follower_num = 0
try:
for follower in me.followers:
L.info(s.log_check_follower.format(
follower.name, follower.id))
if db.is_in_db(conn, follower.id):
L.info(s.log_follower_in_db.format(follower.id))
continue_in_db += 1
else:
L.info(s.log_follower_not_in_db.format(follower.name))
continue_in_db = 0
L.info(s.log_send_message.format(follower.name))
try:
message = calc_message(msg, me, follower,
new_follower_num)
new_follower_num += 1
except Exception as e:
L.exception(e)
message = msg
L.debug(message)
i = 0
while i < 5:
try:
me.send_message(follower, message)
break
except Exception as e:
L.exception(e)
L.debug(s.log_send_failed)
i += 1
else:
L.info(s.log_send_pass)
continue
L.info(s.success)
L.info(s.log_add_user_to_db.format(
follower.name))
db.add_user_to_db(conn, follower)
if continue_in_db == max_old:
L.info(s.log_continue_reach_max.format(max_old))
break
except Exception as e:
L.exception(e)
L.info(s.log_finish_a_pass)
time.sleep(interval)
def run(self, database, msg, interval, log_file, max_old=10):
c.check_type(database, 'database', str)
L = logging.getLogger('qqqfome-backend')
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(message)s')
fh = logging.FileHandler(log_file)
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
sh = logging.StreamHandler()
sh.setLevel(logging.DEBUG)
sh.setFormatter(formatter)
L.setLevel(logging.DEBUG)
L.addHandler(fh)
L.addHandler(sh)
try:
L.info(s.log_connected_to_db.format(database))
conn = db.connect_db(database)
L.info(s.success)
except FileNotFoundError:
L.exception(s.log_file_not_exist.format(database))
L.info(s.exit)
return
# get cookies from database
cookies = db.get_cookies(conn)
if not cookies:
L.exception(s.log_no_cookies_in_database)
L.info(s.exit)
return
L.info(s.log_get_cookies_from_database)
L.debug(cookies)
try:
client = ZhihuClient(cookies)
L.info(s.log_build_zhihu_client)
except Exception as e:
L.exception(e)
return
while True:
L.info(s.log_start_a_pass)
i = 0
while i < 5:
try:
L.info(s.log_build_me)
me = client.me()
break
except Exception as e:
L.exception(e)
i += 1
else:
L.error(s.log_fail_to_build_me)
L.info(s.exit)
return
try:
follower_num = me.follower_num
except Exception as e:
L.exception(e)
L.info(s.log_get_follower_num_failed)
L.info(s.log_finish_a_pass)
time.sleep(interval)
continue
L.info(s.log_get_follower_num.format(follower_num))
db.log_to_db(conn, follower_num, s.log_start_a_pass)
continue_in_db = 0
new_follower_num = 0
try:
for follower in me.followers:
L.info(
s.log_check_follower.format(follower.name,
follower.id))
if db.is_in_db(conn, follower.id):
L.info(s.log_follower_in_db.format(follower.id))
continue_in_db += 1
else:
L.info(s.log_follower_not_in_db.format(follower.name))
continue_in_db = 0
L.info(s.log_send_message.format(follower.name))
try:
message = calc_message(msg, me, follower,
new_follower_num)
new_follower_num += 1
except Exception as e:
L.exception(e)
message = msg
L.debug(message)
i = 0
while i < 5:
try:
me.send_message(follower, message)
break
except Exception as e:
L.exception(e)
L.debug(s.log_send_failed)
i += 1
else:
L.info(s.log_send_pass)
continue
L.info(s.success)
L.info(s.log_add_user_to_db.format(follower.name))
db.add_user_to_db(conn, follower)
if continue_in_db == max_old:
L.info(s.log_continue_reach_max.format(max_old))
break
except Exception as e:
L.exception(e)
L.info(s.log_finish_a_pass)
time.sleep(interval)
import os
import unittest
import shutil
from zhihu import ZhihuClient
from .. import db
file_dir = os.path.dirname(os.path.abspath(__file__))
test_dir = os.path.join(file_dir, 'test')
json_path = os.path.join(file_dir, 'test.json')
author = ZhihuClient(json_path).me()
db_path = os.path.join(test_dir, db.author_to_db_filename(author))
class InitDBTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
os.makedirs(test_dir, exist_ok=True)
os.chdir(test_dir)
def tearDown(self):
self.db.close() if (hasattr(self, 'db') and self.db) else None
try:
os.remove(db_path)
except FileNotFoundError:
pass
@classmethod
def tearDownClass(cls):
shutil.rmtree(test_dir)
#!/usr/bin/env python3
# -*- coding: utf-8 -*
from zhihu import ZhihuClient
#client = ZhihuClient()
#client.set_proxy('http://ucbcn\cnt0095:[email protected]:8080')
#client.login_in_terminal()
#client.create_cookies('cookies.json')
Cookies_File = 'cookies.json'
client = ZhihuClient(Cookies_File)
client.set_proxy('http://ucbcn\cnt0095:[email protected]:8080')
url = 'http://www.zhihu.com/people/douzishushu'
author = client.author(url)
print('用户名 %s' % author.name)
print('用户简介 %s' % author.motto)
print('用户关注人数 %d' % author.followee_num)
print('取用户粉丝数 %d' % author.follower_num)
print('用户得到赞同数 %d' % author.upvote_num)
print('用户得到感谢数 %d' % author.thank_num)
print('用户提问数 %d' % author.question_num)
print('用户答题数 %d' % author.answer_num)
print('用户专栏文章数 %d，名称分别为：' % author.post_num)
for column in author.columns:
print(column.name)
print('用户收藏夹数 %d，名称分别为：' % author.collection_num)
for collection in author.collections:
print(collection.name)
from zhihu import ZhihuClient
Cookies_File = 'cookies.json'
client = ZhihuClient(Cookies_File)
url = 'http://www.zhihu.com/people/excited-vczh'
author = client.author(url)
print('用户名 %s' % author.name)
print('用户简介 %s' % author.motto)
print('用户关注人数 %d' % author.followee_num)
print('取用户粉丝数 %d' % author.follower_num)
print('用户得到赞同数 %d' % author.upvote_num)
print('用户得到感谢数 %d' % author.thank_num)
print('用户提问数 %d' % author.question_num)
print('用户答题数 %d' % author.answer_num)
print('用户专栏文章数 %d，名称分别为：' % author.post_num)
for column in author.columns:
print(column.name)
print('用户收藏夹数 %d，名称分别为：' % author.collection_num)
for collection in author.collections:
print(collection.name)
question = client.question('http://www.zhihu.com/question/28092572')
for answer in question.answers:
answer.save()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import time
import re
from zhihu import ZhihuClient
Cookies_File = 'cookies.json'
client = ZhihuClient(Cookies_File)
# url = 'http://www.zhihu.com/people/excited-vczh'
# author = client.author(url)
#
# print('用户名 %s' % author.name)
# print('用户简介 %s' % author.motto)
# print('用户关注人数 %d' % author.followee_num)
# print('取用户粉丝数 %d' % author.follower_num)
# print('用户得到赞同数 %d' % author.upvote_num)
# print('用户得到感谢数 %d' % author.thank_num)
# print('用户提问数 %d' % author.question_num)
# print('用户答题数 %d' % author.answer_num)
#
# print('用户专栏文章数 %d，名称分别为：' % author.post_num)
# for column in author.columns:
# print(column.name)
# print('用户收藏夹数 %d，名称分别为：' % author.collection_num)
# for collection in author.collections:
# print(collection.name)
# author = client.author('http://www.zhihu.com/people/excited-vczh')
# for act in author.activities:
# if act.type == client.ActType.UPVOTE_ANSWER:
# ==============================
USER_URL = "https://www.zhihu.com/people/7sdream"
FOLLOWER_CHECK_MAX_NUM = 2000
ANSWER_CHECK_MAX_NUM = 20
# ==============================
def is_zero_user(author):
return (author.upvote_num + author.question_num + author.answer_num) <= 3
client = ZhihuClient('test.json')
user = client.author(USER_URL)
print("检查用户{user.name} at {time}".format(user=user, time=datetime.datetime.now()))
if user.follower_num < FOLLOWER_CHECK_MAX_NUM:
FOLLOWER_CHECK_MAX_NUM = user.follower_num
print("正在检查前{FOLLOWER_CHECK_MAX_NUM}个关注者....".format(**locals()))
zeros = 0
for _, follower in zip(range(FOLLOWER_CHECK_MAX_NUM), user.followers):
if is_zero_user(follower):
zeros += 1
