def get_login_client(self):
self.hello_world()
# 先判断图片质量
Config.picture_quality = self.set_picture_quality()
if Config.remember_account:
account = Config.account
password = Config.password
if self.__load_login_client():
# 尝试直接载入token文件
# 载入成功直接返回即可
return self.client
else:
print u"建议直接使用内置帐号登录，敲击回车使用内置帐号，输入任意字符使用自有帐号"
user_input = raw_input()
if not user_input:
account, password = Config.account, Config.password
else:
print u'现在开始登陆流程，请根据提示输入您的账号密码'
print u''
print u''
account, password = self.get_account()
captcha = None
while not self.login(account, password, captcha):
print u'啊哦，登录失败，可能需要输入验证码'
print u'输入『yes』按回车更换其他账号'
print u'直接敲击回车获取验证码'
confirm = raw_input()
if confirm == u'yes':
account, password = self.get_account()
captcha = self.get_captcha()
Config.save()
return self.client
def login(self, account, password, captcha=''):
content = Http.get_content('https://www.zhihu.com/')
xsrf = Match.xsrf(content)
if not xsrf:
Debug.logger.info(u'登陆失败')
Debug.logger.info(u'敲击回车重新发送登陆请求')
return False
xsrf = xsrf.split('=')[1]
# add xsrf as cookie into cookieJar,
cookie = Http.make_cookie(name='_xsrf', value=xsrf, domain='www.zhihu.com')
self.cookieJar.set_cookie(cookie)
if captcha:
post_data = {'_xsrf': xsrf, 'email': account, 'password': password, 'remember_me': True,
'captcha': captcha}
else:
post_data = {'_xsrf': xsrf, 'email': account, 'password': password, 'remember_me': True}
header = {
'Accept': '*/*',
'Accept-Encoding': 'gzip,deflate', # 主要属性，只要有此项知乎即认为来源非脚本
'Accept-Language': 'zh,zh-CN;q=0.8,en-GB;q=0.6,en;q=0.4',
'Host': 'www.zhihu.com',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36(KHTML, like Gecko)Chrome/34.0.1847.116 Safari/537.36',
'Connection': 'keep-alive',
'X-Requested-With': 'XMLHttpRequest',
'Origin': 'https://www.zhihu.com',
'Referer': 'https://www.zhihu.com/',
}
result = Http.get_content(url=r'https://www.zhihu.com/login/email', data=post_data, extra_header=header)
if not result:
Debug.logger.info(u'登陆失败，请敲击回车重新登陆')
return False
response = json.loads(result)
if response['r'] == 0:
print u'登陆成功！'
print u'登陆账号:', account
print u'请问是否需要记住帐号密码？输入yes记住，输入其它任意字符跳过，回车确认'
if raw_input() == 'yes':
Config.account, Config.password, Config.remember_account = account, password, True
print u'帐号密码已保存,可通过修改config.json修改设置'
else:
Config.account, Config.password, Config.remember_account = '', '', False
print u'跳过保存环节，进入下一流程'
Config._save()
cookie = self.get_cookie()
DB.execute('delete from LoginRecord') # 登陆成功后清除数据库中原有的登录记录，避免下次登陆时取到旧记录
data = {}
data['account'] = account
data['password'] = password
data['recordDate'] = ExtraTools.get_today()
data['cookieStr'] = cookie
DB.save(data, 'LoginRecord')
DB.commit()
return True
else:
print u'登陆失败'
Debug.print_dict(response)
return False
def init_config(recipe_kind):
if recipe_kind == 'zhihu': # TODO: 再有一个需要登录的网站, 改掉硬编码
login = Login(recipe_kind='zhihu')
else:
return
# !!!!!发布的时候把Config.remember_account改成false!!!!!,第一次需要登录,之后用cookie即可
# 登陆成功了,自动记录账户
if Config.remember_account_set:
Debug.logger.info(u'检测到有设置文件，直接使用之前的设置')
# if raw_input():
# login.start()
# Config.picture_quality = guide.set_picture_quality()
Config.picture_quality = 1
# else:
try:
Http.set_cookie() # sinablog, jianshu: DontNeed
except TypeError:
print u"没有找到登录成功的cookie记录, 请重新登录"
login.start()
else:
log.warning_log(u"Please login...")
login.start()
# Config.picture_quality = guide.set_picture_quality()
Config.picture_quality = 1
Config.remember_account_set = True
# save config
Config._save()
return
def __init__(self, recipe_kind='Notset', read_list='ReadList.txt', url=None, debug=False):
u"""
配置文件使用$符区隔，同一行内的配置文件归并至一本电子书内
:param recipe_kind:
:param read_list: default value: ReadList.txt
:param url:
:param debug:
:return:
"""
self.recipe_kind = recipe_kind
self.read_list = read_list
self.url = url
log.warning_log(u"website type: " + str(self.recipe_kind) + '\n')
import logging
if debug is True:
Debug.logger.setLevel(logging.DEBUG)
else:
Debug.logger.setLevel(logging.INFO)
Debug.logger.debug(u"read_list: " + str(self.read_list))
Debug.logger.debug(u"url: " + str(self.url))
Debug.logger.debug(u"recipe type:" + str(recipe_kind))
Path.init_base_path(recipe_kind) # 设置路径
Path.init_work_directory() # 创建路径
self.init_database() # 初始化数据库
Config._load()
return
def get_login_client(self):
self.hello_world()
# 先判断图片质量
Config.picture_quality = self.set_picture_quality()
if Config.remember_account:
account = Config.account
password = Config.password
if self.__load_login_client():
# 尝试直接载入token文件
# 载入成功直接返回即可
return self.client
else:
print u"建议直接使用内置帐号登录，敲击回车使用内置帐号，输入任意字符使用自有帐号"
user_input = raw_input()
if not user_input:
account, password = Config.account, Config.password
else:
print u'现在开始登陆流程，请根据提示输入您的账号密码'
print u''
print u''
account, password = self.get_account()
captcha = None
while not self.login(account, password, captcha):
print u'啊哦，登录失败，可能需要输入验证码'
print u'输入『yes』按回车更换其他账号'
print u'直接敲击回车获取验证码'
confirm = raw_input()
if confirm == u'yes':
account, password = self.get_account()
captcha = self.get_captcha()
Config.save()
return self.client
def __init__(self):
u"""
配置文件使用$符区隔，同一行内的配置文件归并至一本电子书内
"""
init.init_database()
Path.init_base_path()
Config._load()
return
def __init__(self):
u"""
配置文件使用$符区隔，同一行内的配置文件归并至一本电子书内
"""
Path.init_base_path() # 设置路径
Path.init_work_directory() # 创建路径
self.init_database() # 初始化数据库
Config._load()
return
def __init__(self):
self.config = Config()
self.framework_folder_root_path = self.config.get_value(
"framework_folder_root_path")
self.local_tar_files_folder_path = self.config.get_value(
"local_tar_files_folder_path")
self.shell_utility = ShellUtility()
self.init_framework_environment()
self.check_framework_environment()
def __init__(self):
# 初始化目录结构
Path.init_base_path()
Path.init_work_directory()
# 初始化数据库链接
DB.init_database()
# 初始化配置
Config.init_config()
return
def __init__(self):
u"""
配置文件使用$符区隔，同一行内的配置文件归并至一本电子书内
"""
Path.init_base_path() # 设置路径
Path.init_work_directory() # 创建路径
self.init_database() # 初始化数据库
Config._load()
return
def __init__(self, ip_list, user_name, password):
self.config = Config()
self.ip_list = ip_list
self.user_name = user_name
self.password = password
self.current_user_name = getpass.getuser()
self.master_ip = self.config.get_value("spark_master_ip")
self.master_hostname = self.config.get_value('master_hostname')
self.local_password = self.config.get_value('local_password')
self.shell_utility = ShellUtility(user_name, password,
self.local_password)
def login_button_clicked(self):
account = str(self.username.text())
password = str(self.password.text())
captcha = str(self.captcha.text())
if not self.login.login(account=account, password=password, captcha=captcha):
click_ok = QtGui.QMessageBox.information(self, u"登陆失败", u"啊哦，登录失败，可能需要输入验证码\n请尝试输入验证码")
if click_ok:
self.login.get_captcha(from_ui=True)
return
Config.remember_account_set = True
Config._save()
QtGui.QMessageBox.information(self, u"登陆成功", u"恭喜, 登陆成功, 登陆信息已经保存")
self.username.setText('')
self.password.setText('')
self.captcha.setText('')
def init_config():
login = Login()
if Config.remember_account:
print u"检测到有设置文件，是否直接使用之前的设置？(帐号、密码、图片质量)"
print u"点按回车使用之前设置，敲入任意字符后点按回车进行重新设置"
if raw_input():
login.start()
Config.picture_quality = guide.set_picture_quality()
else:
Http.set_cookie()
else:
login.start()
Config.picture_quality = guide.set_picture_quality()
# 储存设置
Config._save()
return
def init_config():
login = Login()
if Config.remember_account:
print u'检测到有设置文件，是否直接使用之前的设置？(帐号、密码、图片质量)'
print u'点按回车使用之前设置，敲入任意字符后点按回车进行重新设置'
if raw_input():
login.start()
Config.picture_quality = guide.set_picture_quality()
else:
Http.set_cookie()
else:
login.start()
Config.picture_quality = guide.set_picture_quality()
# 储存设置
Config._save()
return
def login_button_clicked(self):
account = str(self.username.text())
password = str(self.password.text())
captcha = str(self.captcha.text())
if not self.login.login(
account=account, password=password, captcha=captcha):
click_ok = QtGui.QMessageBox.information(
self, u"登陆失败", u"啊哦，登录失败，可能需要输入验证码\n请尝试输入验证码")
if click_ok:
self.login.get_captcha(from_ui=True)
return
Config.remember_account_set = True
Config._save()
QtGui.QMessageBox.information(self, u"登陆成功", u"恭喜, 登陆成功, 登陆信息已经保存")
self.username.setText('')
self.password.setText('')
self.captcha.setText('')
class Deploy(object):
def __init__(self, ip_list, user_name, password):
self.config = Config()
self.ip_list = ip_list
self.user_name = user_name
self.password = password
self.current_user_name = getpass.getuser()
self.master_ip = self.config.get_value("spark_master_ip")
self.master_hostname = self.config.get_value('master_hostname')
self.local_password = self.config.get_value('local_password')
self.shell_utility = ShellUtility(user_name, password,
self.local_password)
def run(self):
pass
def write_to_config_file(self, file_path, contents):
with open(file_path, "w+", encoding="utf-8") as config_file:
for content in contents:
config_file.write(content)
config_file.write("\n")
def login(self, account, password, captcha=None):
if not account or not password:
print u"用户名/密码为空，请先输入用户名/密码"
# 未输入用户名密码，直接返回false
return False
try:
is_login_success, reason = self.client.login(
account, password, captcha)
except NeedCaptchaException:
# 保存验证码并提示输入，重新登录
print u'登录失败，需要输入验证码'
return False
if not is_login_success:
print u'登陆失败'
print u"失败原因 => " + str(reason)
return False
print u'登陆成功！'
print u'登陆账号:', account
# 保持登录token
self.__save_login_client()
if Config.remember_account:
Config.account, Config.password, Config.remember_account = account, password, True
else:
print u'请问是否需要记住帐号密码？输入yes记住，输入其它任意字符跳过，回车确认'
if raw_input() == u'yes':
Config.account, Config.password, Config.remember_account = account, password, True
print u'帐号密码已保存,可通过修改config.json修改设置'
else:
# 清空数据
Config.account, Config.password, Config.remember_account = '', '', False
# 清除token
self.__clear_login_client()
print u'跳过保存环节，进入下一流程'
Config.save()
return True
def login(self, account, password, captcha=None):
if not account or not password:
print u"用户名/密码为空，请先输入用户名/密码"
# 未输入用户名密码，直接返回false
return False
try:
is_login_success, reason = self.client.login(account, password, captcha)
except NeedCaptchaException:
# 保存验证码并提示输入，重新登录
print u'登录失败，需要输入验证码'
return False
if not is_login_success:
print u'登陆失败'
print u"失败原因 => " + str(reason)
return False
print u'登陆成功！'
print u'登陆账号:', account
# 保持登录token
self.__save_login_client()
if Config.remember_account:
Config.account, Config.password, Config.remember_account = account, password, True
else:
print u'请问是否需要记住帐号密码？输入yes记住，输入其它任意字符跳过，回车确认'
if raw_input() == u'yes':
Config.account, Config.password, Config.remember_account = account, password, True
print u'帐号密码已保存,可通过修改config.json修改设置'
else:
# 清空数据
Config.account, Config.password, Config.remember_account = '', '', False
# 清除token
self.__clear_login_client()
print u'跳过保存环节，进入下一流程'
Config.save()
return True
class DistributionFramework(object):
def __init__(self):
self.config = Config()
self.framework_folder_root_path = self.config.get_value(
"framework_folder_root_path")
self.local_tar_files_folder_path = self.config.get_value(
"local_tar_files_folder_path")
self.shell_utility = ShellUtility()
self.init_framework_environment()
self.check_framework_environment()
def init_framework_environment(self):
self.shell_utility.call_shell_command("mkdir -p {}".format(
self.local_tar_files_folder_path))
def check_framework_environment(self):
if not os.path.exists(
self.framework_folder_root_path.format(
username=getpass.getuser())):
raise FileNotFoundError("{} not found.".format(
self.framework_folder_root_path))
if not os.path.exists(self.local_tar_files_folder_path):
raise FileNotFoundError("{} not found.".format(
self.local_tar_files_folder_path))
def __init__(self, task_list):
self.task_set = set(task_list)
self.task_complete_set = set()
self.work_set = set() # 待抓取网址池
self.work_complete_set = set() # 已完成网址池
self.content_list = [] # 用于存放已抓取的内容
self.answer_list = []
self.question_list = []
self.info_list = []
self.extra_index_list = []
self.info_url_set = self.task_set.copy()
self.info_url_complete_set = set()
# 添加扩展属性
self.add_property()
Config._load() # TODO: 可能对性能有影响,改成运行时的环境变量?
if Config.need_account:
Debug.logger.debug("Need cookie")
Http.set_cookie()
else:
Debug.logger.debug("Don't need cookie")
Http.set_cookie('DontNeed')
class FrameworkDeploy(object):
def __init__(self, ip_list, machine_user_account, machine_user_password):
self.ip_list = ip_list
self.machine_user_account = machine_user_account
self.machine_user_password = machine_user_password
self.config = Config()
def deploy(self):
ip_for_ssh_list = list(self.ip_list)
ip_for_ssh_list.append(self.config.get_value("spark_master_ip"))
print(ip_for_ssh_list)
deploies = [
SSHDeploy(ip_for_ssh_list, self.machine_user_account,
self.machine_user_password),
EtcHostsDeploy(self.ip_list, self.machine_user_account,
self.machine_user_password),
HadoopDeploy(self.ip_list, self.machine_user_account,
self.machine_user_password),
SparkDeploy(self.ip_list, self.machine_user_account,
self.machine_user_password)
]
for deploy in deploies:
deploy.run()
def init_config(): # TODO 输出提示语之类的
Config._save()
return
command = "echo \"{}\" | sudo tee --append /etc/hosts".format(
line)
self.shell_utility.run_remote_sudo_command(ip, command)
def generate_hosts_default_file_content(self):
self.hosts_default_content = [
"127.0.0.1\tlocalhost", "::1\tip6-localhost ip6-loopback",
"fe00::0\tip6-localnet", "ff00::0\tip6-mcastprefix",
"ff02::1\tip6-allnodes", "ff02::2\tip6-allrouters"
]
def generate_hosts_client_info_file_content(self):
i = 1
self.hosts_client_info_content = []
print('-' * 50)
print(self.ip_list)
print('-' * 50)
for ip in self.ip_list:
self.hosts_client_info_content.append("{}\t{}".format(
ip, "{}{}".format(self.client_name_prefix, i)))
i = i + 1
if __name__ == "__main__":
config = Config()
ip_list = config.get_value("test_machines_ip_list")
user_name = config.get_value("os_user_name")
password = config.get_value("os_password")
deploy = EtcHostsDeploy(ip_list, user_name, password)
deploy.run()
def init_config():
Config._save()
return
def print_config():
Config._sync()
Debug.print_dict(Config._config_store)
return
def print_config():
Config._sync()
Debug.print_dict(Config._config_store)
return
from functools import partial
from http.server import ThreadingHTTPServer
from os import environ
from signal import signal, SIGTERM, SIGINT
from threading import Thread
from src.components.checker import Checker
from src.components.cleaner import Cleaner
from src.components.controller import Controller
from src.components.creator import Creator
from src.components.listener import Listener
from src.components.resolver import Resolver
from src.components.suicide import Suicide
from src.tools.config import Config
if __name__ == '__main__':
config = Config(environ)
creator = Creator(config)
checker, resolver = Checker(), Resolver(creator)
controller, cleaner = Controller(checker, resolver), Cleaner()
ListenerWithController = partial(Listener, controller, cleaner)
server = ThreadingHTTPServer(config.address, ListenerWithController)
server_thread = Thread(target=server.serve_forever)
server_thread.start()
suicide = Suicide(server)
signal(SIGTERM, suicide.die)
signal(SIGINT, suicide.die)
server_thread.join()
def login(self, account, password, captcha=''):
content = Http.get_content('https://www.zhihu.com/')
xsrf = Match.xsrf(content)
if not xsrf:
Debug.logger.info(u'登陆失败')
Debug.logger.info(u'敲击回车重新发送登陆请求')
return False
xsrf = xsrf.split('=')[1]
# add xsrf as cookie into cookieJar,
cookie = Http.make_cookie(name='_xsrf',
value=xsrf,
domain='www.zhihu.com')
self.cookieJar.set_cookie(cookie)
if captcha:
post_data = {
'_xsrf': xsrf,
'email': account,
'password': password,
'remember_me': True,
'captcha': captcha
}
else:
post_data = {
'_xsrf': xsrf,
'email': account,
'password': password,
'remember_me': True
}
header = {
'Accept': '*/*',
'Accept-Encoding': 'gzip,deflate', # 主要属性，只要有此项知乎即认为来源非脚本
'Accept-Language': 'zh,zh-CN;q=0.8,en-GB;q=0.6,en;q=0.4',
'Host': 'www.zhihu.com',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36(KHTML, like Gecko)Chrome/34.0.1847.116 Safari/537.36',
'Connection': 'keep-alive',
'X-Requested-With': 'XMLHttpRequest',
'Origin': 'https://www.zhihu.com',
'Referer': 'https://www.zhihu.com/',
}
result = Http.get_content(url=r'https://www.zhihu.com/login/email',
data=post_data,
extra_header=header)
if not result:
Debug.logger.info(u'登陆失败，请敲击回车重新登陆')
return False
response = json.loads(result)
if response['r'] == 0:
print u'登陆成功！'
print u'登陆账号:', account
print u'请问是否需要记住帐号密码？输入yes记住，输入其它任意字符跳过，回车确认'
if raw_input() == 'yes':
Config.account, Config.password, Config.remember_account = account, password, True
print u'帐号密码已保存,可通过修改config.json修改设置'
else:
Config.account, Config.password, Config.remember_account = '', '', False
print u'跳过保存环节，进入下一流程'
Config._save()
cookie = self.get_cookie()
DB.execute(
'delete from LoginRecord') # 登陆成功后清除数据库中原有的登录记录，避免下次登陆时取到旧记录
data = {}
data['account'] = account
data['password'] = password
data['recordDate'] = ExtraTools.get_today()
data['cookieStr'] = cookie
DB.save(data, 'LoginRecord')
DB.commit()
return True
else:
print u'登陆失败'
Debug.print_dict(response)
return False
def init_config(): # TODO 输出提示语之类的
Config._save()
return
def init_config():
Config._save()
return
def __init__(self, ip_list, machine_user_account, machine_user_password):
self.ip_list = ip_list
self.machine_user_account = machine_user_account
self.machine_user_password = machine_user_password
self.config = Config()
