def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
self.db = RedisClient().db
# 哈希表来统计拨号VPS的IP
if not self.db.hexists('dialed_IPs', proxy):
self.db.hset('dialed_IPs', proxy, 1)
# 往IP池里插入数据
if self.redis.set(CLIENT_NAME, proxy):
logger.info(f'Successfully set proxy {proxy}')
return True
else:
num = int(self.db.hget('dialed_IPs', proxy))
logger.info(f'{proxy} in proxy pools {num} times already')
if num <2:
self.db.hset('dialed_IPs', proxy, num+1)
# 往IP池里插入数据
if self.redis.set(CLIENT_NAME, proxy):
logger.info(f'Successfully set proxy {proxy}')
return True
else:
return False
def adsl(self):
"""
拨号主进程
:return: None
"""
while True:
self.log.info(
"ADSL is Starting, And Removing Proxy, Please wait......")
self.log.addHandler(self.log_file_handler)
try:
self.remove_proxy()
self.redis = None
except:
while True:
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
self.redis = RedisClient()
self.remove_proxy()
self.redis = None
break
self.redis = None
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
self.redis = RedisClient()
if status == 0:
self.log.info("ADSL Successfully")
self.log.addHandler(self.log_file_handler)
ip = self.get_ip()
if ip:
self.log.info("The new IP is : %s" % ip)
self.log.info("Testing Proxy, Please Wait")
self.log.addHandler(self.log_file_handler)
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
if self.test_proxy(proxy):
self.log.info("This is a valid proxy")
self.set_proxy(proxy)
self.log.info('The program is sleeping for %s second' %
ADSL_CYCLE)
self.log.addHandler(self.log_file_handler)
time.sleep(ADSL_CYCLE)
else:
self.log.warning("Oops.This is a invalid proxy")
self.log.addHandler(self.log_file_handler)
else:
self.log.warning("Get IP Failed, Re Dialing")
self.log.addHandler(self.log_file_handler)
time.sleep(ADSL_ERROR_CYCLE)
else:
self.log.warning("Oops.ADSL Failed, Please Check")
self.log.addHandler(self.log_file_handler)
time.sleep(ADSL_ERROR_CYCLE)
def remove_proxy(self):
"""
移除代理
:return: None
"""
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('已移除该代理IP')
def remove_proxy(self):
"""
移除代理
:return: None
"""
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('已设置代理IP', proxy)
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
logger.info(f'Successfully set proxy {proxy}')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
def serve(redis=None, port=SERVER_PORT, address=SERVER_HOST):
if not redis:
redis = RedisClient()
application = Application([
(r'/', Server, dict(redis=redis)),
(r'/(.*)', Server, dict(redis=redis)),
])
application.listen(port, address=address)
logger.info(f'API Listening on http://{address}:{port}')
tornado.ioloop.IOLoop.instance().start()
def solve_badhosts():
# 处理问题主机，根据问题类型，重装软件或拨号。
badhosts_info_dict = RedisClient(list_key='badhosts').all()
if badhosts_info_dict:
for key, v in badhosts_info_dict.items():
values = v
# 从配置文件读取（链接信息可能已被修正了）
for _group in HOSTS_GROUP:
if key in HOSTS_GROUP.get(_group).keys():
values = HOSTS_GROUP.get(_group)[key]
with paramiko.SSHClient() as ssh_cli:
ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh_cli.connect(hostname=values['host'], username=values['username'],
password=values['password'],
port=values['port'])
except paramiko.ssh_exception.NoValidConnectionsError as e:
# init_error错误是ssh信息配置错误，只要修正了配置文件就能继续下去。
# 如果还是链接不上，还是init_error则更新信息。
logger.error(f'主机：{key}，配置有问题，请手动修复并重启程序! {e}')
values['problem'] = 'init_error'
RedisClient(list_key='badhosts').set(key, json.dumps(values))
# 如果是adsl_error错误，则需要重新安装软件
if values['problem'] == 'adsl_error':
clean_sys(ssh_cli)
check_host(ssh_cli)
# 开始拨号
try:
proxy_ip = pppoe(ssh_cli, values['cmd'])
except Exception:
# 依然有问题，不做操作
pass
# 如果没问题，加入代理，从问题主机列表移除并添加到正常主机列表
RedisClient(list_key='adslproxy').set(key, f'{proxy_ip}:{PROXY_PORT}')
RedisClient(list_key='badhosts').remove(key)
RedisClient(list_key='goodhosts').set(key, json.dumps(values))
# 间隔300秒 时间再次执行
threading.Timer(300, solve_badhosts).start()
def remove_proxy(self):
"""
移除代理
:return: None
"""
try:
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
except:
print('nothing to remove')
def run_task(key, values):
# print(threading.currentThread().getName())
print('初始化主机：', key)
with paramiko.SSHClient() as ssh_cli:
ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh_cli.connect(hostname=values['host'],
username=values['username'],
password=values['password'],
port=values['port'])
except paramiko.ssh_exception.NoValidConnectionsError as e:
logger.error(f'主机：{key}，配置有问题，请手动修复并重启程序! {e}')
values['problem'] = 'init_error'
RedisClient(list_key='badhosts').set(key, json.dumps(values))
return False
# 检查squid
check_host(ssh_cli)
# 开始拨号
proxy_ip = pppoe(ssh_cli, values['cmd'])
# 存储到Redis
RedisClient(list_key='adslproxy').set(key, f'{proxy_ip}:{PROXY_PORT}')
RedisClient(list_key='goodhosts').set(key, json.dumps(values))
def login():
"""
客户端发送json过来
{
"username":"******",
"password":"******"
}
"""
if request.headers['Content-Type'] != 'application/json':
return jsonify({"status": "400", 'error': '请使用 Json 格式传递用户名和密码'}), 500
json_data = request.get_json()
username = json_data.get('username')
password = json_data.get('password')
if username is None or password is None:
abort(400)
# 这里校验账户是否合法，我这里用来redis简单对比；关系型数据库需要自行修改。
# 这里使用了redis做AB数据集切换（账户密码是定时从配置文件读取并更新的），redis方法是自己封装的。
list_key = RedisClient(list_key='ab_set').get('a_or_b')
if RedisClient(list_key=list_key).get(username) == password:
token = genTokenSeq(username)
return jsonify(token)
else:
return jsonify({"status": "500", 'error': '请传递配置文件中正确的用户名和密码'}), 500
def adsl_switch_ip():
# 定时拨号的主机是从正常的主机中获取的。
hosts_info_dict = RedisClient(list_key='goodhosts').all()
# 开始拨号（从拨号到IP可用有一定时间间隔，不要用异步，防止短时间内无IP可用）
for key, values in hosts_info_dict.items():
with paramiko.SSHClient() as ssh_cli:
ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# 这里不用捕捉异常，在hosts_init()时即可判断时否异常
ssh_cli.connect(hostname=values['host'], username=values['username'],
password=values['password'],
port=values['port'])
try:
proxy_ip = pppoe(ssh_cli, values['cmd'])
except Exception:
# 重新拨号得不到新IP，则移除旧IP，且从正常主机列表移除并加入问题主机列表
logger.error(f'{key}:拨号失败了！')
RedisClient(list_key='adslproxy').remove(key)
RedisClient(list_key='goodhosts').remove(key)
values['problem'] = 'adsl_error'
RedisClient(list_key='badhosts').set(key, json.dumps(values))
# 存储到Redis
RedisClient(list_key='adslproxy').set(key, f'{proxy_ip}:{PROXY_PORT}')
# 间隔ADSL_SWITCH_TIME 时间再次执行
threading.Timer(ADSL_SWITCH_TIME, adsl_switch_ip).start()
def remove_proxy(self):
"""
移除代理
:return: None
"""
logger.info(f'Removing {CLIENT_NAME}...')
try:
# 由于拨号就会中断连接，所以每次都要重新建立连接
if hasattr(self, 'redis') and self.redis:
self.redis.close()
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
logger.info(f'Removed {CLIENT_NAME} successfully')
return True
except redis.ConnectionError:
logger.info(f'Remove {CLIENT_NAME} failed')
def update_user_info():
# 更新API用户信息(AB集切换)
a_or_b = RedisClient(list_key='ab_set').get('a_or_b')
if a_or_b == "userinfo_a":
for group in USER:
for user_info in USER[group].values():
RedisClient(list_key='userinfo_b').set(user_info['username'], user_info['password'])
RedisClient(list_key='ab_set').set('a_or_b', 'userinfo_b')
RedisClient(list_key='userinfo_a').delete()
elif a_or_b == "userinfo_b":
for group in USER:
for user_info in USER[group].values():
RedisClient(list_key='userinfo_a').set(user_info['username'], user_info['password'])
RedisClient(list_key='ab_set').set('a_or_b', 'userinfo_a')
RedisClient(list_key='userinfo_b').delete()
# 间隔60秒 时间再次执行
threading.Timer(60, update_user_info).start()
def remove_proxy(self):
"""
移除代理
:return: None
通常情况下，连续拨号失败几次就需要重启机器了，这时候VPS已经无法成功拨号连接互联网了
"""
logger.info(f'Removing {CLIENT_NAME}...')
try:
# 由于拨号就会中断连接，所以每次都要重新建立连接
if hasattr(self, 'redis') and self.redis:
self.redis.close()
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
logger.info(f'Removed {CLIENT_NAME} successfully')
return True
except redis.ConnectionError:
logger.info(f'Remove {CLIENT_NAME} failed')
logger.error('删除IP失败!从代理池删除IP并重启系统.......')
os.system('/usr/sbin/shutdown -r now')
def __init__(self):
self.redis = RedisClient()
# 日志打印格式
log_fmt = '%(asctime)s\tFile \"%(filename)s\",line %(lineno)s\t%(levelname)s: %(message)s'
formatter = logging.Formatter(log_fmt)
# 创建TimedRotatingFileHandler对象
self.log_file_handler = TimedRotatingFileHandler(filename="adsl.log",
when="D",
interval=1,
backupCount=7)
self.log_file_handler.suffix = "%Y-%m-%d_%H-%M.log"
self.log_file_handler.extMatch = re.compile(
r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}.log$")
self.log_file_handler.setFormatter(formatter)
logging.basicConfig(level=logging.INFO)
self.log = logging.getLogger()
self.log.addHandler(self.log_file_handler)
RedisClient(list_key='userinfo_b').delete()
# 间隔60秒 时间再次执行
threading.Timer(60, update_user_info).start()
def tasks_main():
# hosts_manages启动时会初始化主机，并把代理写入Redis，此处接着执行定时任务即可。
t2 = threading.Timer(ADSL_SWITCH_TIME, adsl_switch_ip)
t2.start()
# 立刻处理问题主机
# print("处理问题主机！")
t3 = threading.Timer(0, solve_badhosts)
t3.start()
if __name__ == "__main__":
# 清空Redis中的数据
RedisClient(list_key='adslproxy').delete()
RedisClient(list_key='goodhosts').delete()
RedisClient(list_key='badhosts').delete()
RedisClient(list_key='userinfo_a').delete()
RedisClient(list_key='userinfo_b').delete()
# 定时更新用户账户
RedisClient(list_key='ab_set').set('a_or_b', 'userinfo_b') # 启动初始化要设置一下，防止key不存在报错。
t1 = threading.Timer(0, update_user_info)
t1.start()
# hosts_init启动时会初始化主机，并把代理写入Redis，此处接着执行定时任务即可。
hosts_init() # join线程阻塞（配置环境需要时间，只花最慢一台机器的时间）
# 开始定时拨号任务，子线程开始等待，不影响下面执行
tasks_main()
def remove_proxy(self):
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
def __init__(self):
self.db = RedisClient()
self.counts = defaultdict(int)
# coding=utf-8
from adslproxy.api import server
from adslproxy.db import RedisClient
from adslproxy.config import *
if __name__ == '__main__':
redis = RedisClient(host=REDIS_HOST, password=REDIS_PASSWORD)
server(redis=redis)
# coding=utf-8
from adslproxy.api import server
from adslproxy.db import RedisClient
import os
passwd = os.getenv('REDISPASSWORD')
if __name__ == '__main__':
redis = RedisClient(host='', password=passwd)
server(redis=redis, port=8000)
def set_proxy(self, proxy):
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
# 获取当前文件的上级目录(项目根目录)
WORK_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(WORK_DIR)
sys.path.append(os.path.join(WORK_DIR, 'adslproxy'))
sys.path.append(os.path.join(WORK_DIR, 'config'))
import time
import wrapt
from adslproxy.db import RedisClient
from config.api_config import *
from itsdangerous import TimedJSONWebSignatureSerializer as Serializer
from itsdangerous import SignatureExpired, BadSignature, BadData
from flask import Flask, jsonify, request, abort, url_for
app = Flask(__name__)
redis_cli = RedisClient(list_key='adslproxy')
#############################################################
def genTokenSeq(user):
"""
# 生成token
:param user: 输入用户名
:return: 两个token
"""
access_token_gen = Serializer(secret_key=SECRET_KEY,
salt=SALT,
expires_in=ACCESS_TOKEN_EXPIRES)
refresh_token_gen = Serializer(secret_key=SECRET_KEY,
salt=SALT,
from adslproxy.api import server
from adslproxy.db import RedisClient
if __name__ == '__main__':
redis = RedisClient()
server(redis=redis)
#coding:utf-8
from adslproxy.api import server
from adslproxy.db import RedisClient
if __name__ == '__main__':
redis = RedisClient(host='114.116.13.33', password='******')
server(redis=redis)
# redis = RedisClient(host='114.116.13.33',password='******')
# random = redis.random()
# print(random)