def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ self.redis = RedisClient() self.db = RedisClient().db # 哈希表来统计拨号VPS的IP if not self.db.hexists('dialed_IPs', proxy): self.db.hset('dialed_IPs', proxy, 1) # 往IP池里插入数据 if self.redis.set(CLIENT_NAME, proxy): logger.info(f'Successfully set proxy {proxy}') return True else: num = int(self.db.hget('dialed_IPs', proxy)) logger.info(f'{proxy} in proxy pools {num} times already') if num <2: self.db.hset('dialed_IPs', proxy, num+1) # 往IP池里插入数据 if self.redis.set(CLIENT_NAME, proxy): logger.info(f'Successfully set proxy {proxy}') return True else: return False
class Sender(): def get_ip(self, ifname=ADSL_IFNAME): (status, output) = subprocess.getstatusoutput('ifconfig') if status == 0: pattern = re.compile( ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S) result = re.search(pattern, output) if result: ip = result.group(1) return ip def test_proxy(self, proxy): try: response = requests.get(TEST_URL, proxies={ 'http': 'http://' + proxy, 'https': 'https://' + proxy }, timeout=TEST_TIMEOUT) if response.status_code == 200: return True except (ConnectionError, ReadTimeout): return False def remove_proxy(self): self.redis = RedisClient() self.redis.remove(CLIENT_NAME) print('Successfully Removed Proxy') def set_proxy(self, proxy): self.redis = RedisClient() if self.redis.set(CLIENT_NAME, proxy): print('Successfully Set Proxy', proxy) def adsl(self): while True: print('ADSL Start, Remove Proxy, Please wait') self.remove_proxy() (status, output) = subprocess.getstatusoutput(ADSL_BASH) if status == 0: print('ADSL Successfully') ip = self.get_ip() if ip: print('Now IP', ip) print('Testing Proxy, Please Wait') proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT) if self.test_proxy(proxy): print('Valid Proxy') self.set_proxy(proxy) print('Sleeping') time.sleep(ADSL_CYCLE) else: print('Invalid Proxy') else: print('Get IP Failed, Re Dialing') time.sleep(ADSL_ERROR_CYCLE) else: print('ADSL Failed, Please Check') time.sleep(ADSL_ERROR_CYCLE)
def remove_proxy(self): """ 移除代理 :return: None """ self.redis = RedisClient() self.redis.remove(CLIENT_NAME) print('Successfully Removed Proxy')
def remove_proxy(self): """ 移除代理 :return: None """ self.redis = RedisClient() self.redis.remove(CLIENT_NAME) print('已移除该代理IP')
def adsl(self): """ 拨号主进程 :return: None """ while True: self.log.info( "ADSL is Starting, And Removing Proxy, Please wait......") self.log.addHandler(self.log_file_handler) try: self.remove_proxy() self.redis = None except: while True: (status, output) = subprocess.getstatusoutput(ADSL_BASH) if status == 0: self.redis = RedisClient() self.remove_proxy() self.redis = None break self.redis = None (status, output) = subprocess.getstatusoutput(ADSL_BASH) self.redis = RedisClient() if status == 0: self.log.info("ADSL Successfully") self.log.addHandler(self.log_file_handler) ip = self.get_ip() if ip: self.log.info("The new IP is : %s" % ip) self.log.info("Testing Proxy, Please Wait") self.log.addHandler(self.log_file_handler) proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT) if self.test_proxy(proxy): self.log.info("This is a valid proxy") self.set_proxy(proxy) self.log.info('The program is sleeping for %s second' % ADSL_CYCLE) self.log.addHandler(self.log_file_handler) time.sleep(ADSL_CYCLE) else: self.log.warning("Oops.This is a invalid proxy") self.log.addHandler(self.log_file_handler) else: self.log.warning("Get IP Failed, Re Dialing") self.log.addHandler(self.log_file_handler) time.sleep(ADSL_ERROR_CYCLE) else: self.log.warning("Oops.ADSL Failed, Please Check") self.log.addHandler(self.log_file_handler) time.sleep(ADSL_ERROR_CYCLE)
def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ self.redis = RedisClient() if self.redis.set(CLIENT_NAME, proxy): print('Successfully Set Proxy', proxy)
def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ self.redis = RedisClient() if self.redis.set(CLIENT_NAME, proxy): print('已设置代理IP', proxy)
def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ self.redis = RedisClient() if self.redis.set(CLIENT_NAME, proxy): logger.info(f'Successfully set proxy {proxy}')
class Checker(object): def __init__(self): self.db = RedisClient() self.counts = defaultdict(int) def check(self, proxy): """ 测试代理,返回测试结果 :param proxy: 代理 :return: 测试结果 """ try: response = requests.get(settings.TEST_URL, proxies={ 'http': 'http://' + proxy, 'https': 'https://' + proxy }, timeout=settings.TEST_TIMEOUT) logger.debug(f'Using {proxy} to test {settings.TEST_URL}...') if response.status_code == 200: return True except (ConnectionError, ReadTimeout): return False def run(self): """ 测试一轮 :return: """ proxies = self.db.all() logger.info(f'Try to get all proxies {proxies}') for name, proxy in proxies.items(): # 检测无效 if not self.check(proxy): logger.info(f'Proxy {proxy} invalid') self.counts[proxy] += 1 else: logger.info(f'Proxy {proxy} valid') count = self.counts.get(proxy) or 0 logger.debug( f'Count {count}, TEST_MAX_ERROR_COUNT {settings.TEST_MAX_ERROR_COUNT}' ) if count >= settings.TEST_MAX_ERROR_COUNT: self.db.remove(name) def loop(self): """ 循环测试 :return: """ while True: logger.info('Check for infinite') self.run() logger.info(f'Tested, sleeping for {settings.TEST_CYCLE}s...') time.sleep(settings.TEST_CYCLE)
def remove_proxy(self): """ 移除代理 :return: None """ try: self.redis = RedisClient() self.redis.remove(CLIENT_NAME) print('Successfully Removed Proxy') except: print('nothing to remove')
def remove_proxy(self): """ 移除代理 :return: None """ logger.info(f'Removing {CLIENT_NAME}...') try: # 由于拨号就会中断连接,所以每次都要重新建立连接 if hasattr(self, 'redis') and self.redis: self.redis.close() self.redis = RedisClient() self.redis.remove(CLIENT_NAME) logger.info(f'Removed {CLIENT_NAME} successfully') return True except redis.ConnectionError: logger.info(f'Remove {CLIENT_NAME} failed')
def serve(redis=None, port=SERVER_PORT, address=SERVER_HOST): if not redis: redis = RedisClient() application = Application([ (r'/', Server, dict(redis=redis)), (r'/(.*)', Server, dict(redis=redis)), ]) application.listen(port, address=address) logger.info(f'API Listening on http://{address}:{port}') tornado.ioloop.IOLoop.instance().start()
def remove_proxy(self): """ 移除代理 :return: None 通常情况下,连续拨号失败几次就需要重启机器了,这时候VPS已经无法成功拨号连接互联网了 """ logger.info(f'Removing {CLIENT_NAME}...') try: # 由于拨号就会中断连接,所以每次都要重新建立连接 if hasattr(self, 'redis') and self.redis: self.redis.close() self.redis = RedisClient() self.redis.remove(CLIENT_NAME) logger.info(f'Removed {CLIENT_NAME} successfully') return True except redis.ConnectionError: logger.info(f'Remove {CLIENT_NAME} failed') logger.error('删除IP失败!从代理池删除IP并重启系统.......') os.system('/usr/sbin/shutdown -r now')
def __init__(self): self.redis = RedisClient() # 日志打印格式 log_fmt = '%(asctime)s\tFile \"%(filename)s\",line %(lineno)s\t%(levelname)s: %(message)s' formatter = logging.Formatter(log_fmt) # 创建TimedRotatingFileHandler对象 self.log_file_handler = TimedRotatingFileHandler(filename="adsl.log", when="D", interval=1, backupCount=7) self.log_file_handler.suffix = "%Y-%m-%d_%H-%M.log" self.log_file_handler.extMatch = re.compile( r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}.log$") self.log_file_handler.setFormatter(formatter) logging.basicConfig(level=logging.INFO) self.log = logging.getLogger() self.log.addHandler(self.log_file_handler)
def run_task(key, values): # print(threading.currentThread().getName()) print('初始化主机:', key) with paramiko.SSHClient() as ssh_cli: ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh_cli.connect(hostname=values['host'], username=values['username'], password=values['password'], port=values['port']) except paramiko.ssh_exception.NoValidConnectionsError as e: logger.error(f'主机:{key},配置有问题,请手动修复并重启程序! {e}') values['problem'] = 'init_error' RedisClient(list_key='badhosts').set(key, json.dumps(values)) return False # 检查squid check_host(ssh_cli) # 开始拨号 proxy_ip = pppoe(ssh_cli, values['cmd']) # 存储到Redis RedisClient(list_key='adslproxy').set(key, f'{proxy_ip}:{PROXY_PORT}') RedisClient(list_key='goodhosts').set(key, json.dumps(values))
def login(): """ 客户端发送json过来 { "username":"******", "password":"******" } """ if request.headers['Content-Type'] != 'application/json': return jsonify({"status": "400", 'error': '请使用 Json 格式传递用户名和密码'}), 500 json_data = request.get_json() username = json_data.get('username') password = json_data.get('password') if username is None or password is None: abort(400) # 这里校验账户是否合法,我这里用来redis简单对比;关系型数据库需要自行修改。 # 这里使用了redis做AB数据集切换(账户密码是定时从配置文件读取并更新的),redis方法是自己封装的。 list_key = RedisClient(list_key='ab_set').get('a_or_b') if RedisClient(list_key=list_key).get(username) == password: token = genTokenSeq(username) return jsonify(token) else: return jsonify({"status": "500", 'error': '请传递配置文件中正确的用户名和密码'}), 500
def update_user_info(): # 更新API用户信息(AB集切换) a_or_b = RedisClient(list_key='ab_set').get('a_or_b') if a_or_b == "userinfo_a": for group in USER: for user_info in USER[group].values(): RedisClient(list_key='userinfo_b').set(user_info['username'], user_info['password']) RedisClient(list_key='ab_set').set('a_or_b', 'userinfo_b') RedisClient(list_key='userinfo_a').delete() elif a_or_b == "userinfo_b": for group in USER: for user_info in USER[group].values(): RedisClient(list_key='userinfo_a').set(user_info['username'], user_info['password']) RedisClient(list_key='ab_set').set('a_or_b', 'userinfo_a') RedisClient(list_key='userinfo_b').delete() # 间隔60秒 时间再次执行 threading.Timer(60, update_user_info).start()
def solve_badhosts(): # 处理问题主机,根据问题类型,重装软件或拨号。 badhosts_info_dict = RedisClient(list_key='badhosts').all() if badhosts_info_dict: for key, v in badhosts_info_dict.items(): values = v # 从配置文件读取(链接信息可能已被修正了) for _group in HOSTS_GROUP: if key in HOSTS_GROUP.get(_group).keys(): values = HOSTS_GROUP.get(_group)[key] with paramiko.SSHClient() as ssh_cli: ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh_cli.connect(hostname=values['host'], username=values['username'], password=values['password'], port=values['port']) except paramiko.ssh_exception.NoValidConnectionsError as e: # init_error错误是ssh信息配置错误,只要修正了配置文件就能继续下去。 # 如果还是链接不上,还是init_error则更新信息。 logger.error(f'主机:{key},配置有问题,请手动修复并重启程序! {e}') values['problem'] = 'init_error' RedisClient(list_key='badhosts').set(key, json.dumps(values)) # 如果是adsl_error错误,则需要重新安装软件 if values['problem'] == 'adsl_error': clean_sys(ssh_cli) check_host(ssh_cli) # 开始拨号 try: proxy_ip = pppoe(ssh_cli, values['cmd']) except Exception: # 依然有问题,不做操作 pass # 如果没问题,加入代理,从问题主机列表移除并添加到正常主机列表 RedisClient(list_key='adslproxy').set(key, f'{proxy_ip}:{PROXY_PORT}') RedisClient(list_key='badhosts').remove(key) RedisClient(list_key='goodhosts').set(key, json.dumps(values)) # 间隔300秒 时间再次执行 threading.Timer(300, solve_badhosts).start()
def adsl_switch_ip(): # 定时拨号的主机是从正常的主机中获取的。 hosts_info_dict = RedisClient(list_key='goodhosts').all() # 开始拨号(从拨号到IP可用有一定时间间隔,不要用异步,防止短时间内无IP可用) for key, values in hosts_info_dict.items(): with paramiko.SSHClient() as ssh_cli: ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # 这里不用捕捉异常,在hosts_init()时即可判断时否异常 ssh_cli.connect(hostname=values['host'], username=values['username'], password=values['password'], port=values['port']) try: proxy_ip = pppoe(ssh_cli, values['cmd']) except Exception: # 重新拨号得不到新IP,则移除旧IP,且从正常主机列表移除并加入问题主机列表 logger.error(f'{key}:拨号失败了!') RedisClient(list_key='adslproxy').remove(key) RedisClient(list_key='goodhosts').remove(key) values['problem'] = 'adsl_error' RedisClient(list_key='badhosts').set(key, json.dumps(values)) # 存储到Redis RedisClient(list_key='adslproxy').set(key, f'{proxy_ip}:{PROXY_PORT}') # 间隔ADSL_SWITCH_TIME 时间再次执行 threading.Timer(ADSL_SWITCH_TIME, adsl_switch_ip).start()
from adslproxy.api import server from adslproxy.db import RedisClient if __name__ == '__main__': redis = RedisClient() server(redis=redis)
class Sender(object): """ 拨号并发送到 Redis """ def extract_ip(self): """ 获取本机IP :param ifname: 网卡名称 :return: """ (status, output) = subprocess.getstatusoutput('ifconfig') if not status == 0: return pattern = re.compile(DIAL_IFNAME + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S) result = re.search(pattern, output) if result: # 返回拨号后的 IP 地址 return result.group(1) def test_proxy(self, proxy): """ 测试代理,返回测试结果 :param proxy: 代理 :return: 测试结果 """ try: response = requests.get(TEST_URL, proxies={ 'http': 'http://' + proxy, 'https': 'https://' + proxy }, timeout=TEST_TIMEOUT) if response.status_code == 200: return True except (ConnectionError, ReadTimeout): return False @retry(retry_on_result=lambda x: x is not True, stop_max_attempt_number=10) def remove_proxy(self): """ 移除代理 :return: None """ logger.info(f'Removing {CLIENT_NAME}...') try: # 由于拨号就会中断连接,所以每次都要重新建立连接 if hasattr(self, 'redis') and self.redis: self.redis.close() self.redis = RedisClient() self.redis.remove(CLIENT_NAME) logger.info(f'Removed {CLIENT_NAME} successfully') return True except redis.ConnectionError: logger.info(f'Remove {CLIENT_NAME} failed') def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ self.redis = RedisClient() if self.redis.set(CLIENT_NAME, proxy): logger.info(f'Successfully set proxy {proxy}') def loop(self): """ 循环拨号 :return: """ while True: logger.info('Starting dial...') self.run() time.sleep(DIAL_CYCLE) def run(self): """ 拨号主进程 :return: None """ logger.info('Dial started, remove proxy') try: self.remove_proxy() except RetryError: logger.error('Retried for max times, continue') # 拨号 (status, output) = subprocess.getstatusoutput(DIAL_BASH) if not status == 0: logger.error('Dial failed') # 获取拨号 IP ip = self.extract_ip() if ip: logger.info(f'Get new IP {ip}') if PROXY_USERNAME and PROXY_PASSWORD: proxy = '{username}:{password}@{ip}:{port}'.format(username=PROXY_USERNAME, password=PROXY_PASSWORD, ip=ip, port=PROXY_PORT) else: proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT) time.sleep(10) if self.test_proxy(proxy): logger.info(f'Valid proxy {proxy}') # 将代理放入数据库 self.set_proxy(proxy) time.sleep(DIAL_CYCLE) else: logger.error(f'Proxy invalid {proxy}') else: # 获取 IP 失败,重新拨号 logger.error('Get IP failed, re-dialing') self.run()
class Sender(): def get_ip(self, ifname=ADSL_IFNAME): """ 获取本机IP :param ifname: 网卡名称 :return: """ (status, output) = subprocess.getstatusoutput('ifconfig') if status == 0: pattern = re.compile(ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S) result = re.search(pattern, output) if result: ip = result.group(1) return ip def test_proxy(self, proxy): """ 测试代理 :param proxy: 代理 :return: 测试结果 """ try: headers = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" } response = requests.get(TEST_URL, headers=headers, proxies={ 'http': 'http://' + proxy, 'https': 'https://' + proxy }, timeout=TEST_TIMEOUT) if response.status_code == 200: return True except (ConnectionError, ReadTimeout): return False def remove_proxy(self): """ 移除代理 :return: None """ self.redis = RedisClient() self.redis.remove(CLIENT_NAME) print('Successfully Removed Proxy') def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ self.redis = RedisClient() if self.redis.set(CLIENT_NAME, proxy): print('Successfully Set Proxy', proxy) def adsl(self): """ 拨号主进程 :return: None """ while True: print('ADSL Start, Remove Proxy, Please wait') try: self.remove_proxy() except: while True: # (status, output) = subprocess.getstatusoutput(ADSL_BASH) (status, output) = subprocess.getstatusoutput(ADSL_BASH_STOP) if status == 0: print("adsl-start after {} seconds".format(ADSL_STEP)) time.sleep(ADSL_STEP) (status, output) = subprocess.getstatusoutput(ADSL_BASH_START) if status == 0: self.remove_proxy() break # adsl-stop;adsl-start间隔太短会造成IP不能改变 # (status, output) = subprocess.getstatusoutput(ADSL_BASH) (status, output) = subprocess.getstatusoutput(ADSL_BASH_STOP) if status == 0: print("adsl-start after {} seconds".format(ADSL_STEP)) time.sleep(ADSL_STEP) (status, output) = subprocess.getstatusoutput(ADSL_BASH_START) if status == 0: print('ADSL Successfully') ip = self.get_ip() if ip: print('Now IP', ip) print('Testing Proxy, Please Wait') proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT) if self.test_proxy(proxy): print('Valid Proxy') self.set_proxy(proxy) print('Sleeping') pbar = tqdm(total=ADSL_CYCLE) for i in range(int(ADSL_CYCLE/2)): time.sleep(2) pbar.update(2) pbar.close() # time.sleep(ADSL_CYCLE) else: print('Invalid Proxy') else: print('Get IP Failed, Re Dialing') time.sleep(ADSL_ERROR_CYCLE) else: print('ADSL Failed, Please Check') time.sleep(ADSL_ERROR_CYCLE)
class Sender(): def get_ip(self, ifname=ADSL_IFNAME): """ 获取本机IP :param ifname: 网卡名称 :return: """ (status, output) = subprocess.getstatusoutput('ifconfig') if status == 0: pattern = re.compile( ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S) result = re.search(pattern, output) if result: ip = result.group(1) return ip def test_proxy(self, proxy): """ 测试代理 :param proxy: 代理 :return: 测试结果 """ headers = { 'Accept': 'text/html, application/xhtml+xml, image/jxr, */*', 'Accept - Encoding': 'gzip, deflate', 'Accept-Language': 'zh-Hans-CN, zh-Hans; q=0.5', 'Connection': 'Keep-Alive', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063' } try: response = requests.get(TEST_URL, headers=headers, proxies={ 'http': 'http://' + proxy, 'https': 'https://' + proxy }, timeout=TEST_TIMEOUT) if response.status_code == 200: return True else: print('response_code is', response.status_code) return False except (ConnectionError, ReadTimeout): return False def remove_proxy(self): """ 移除代理 :return: None """ try: self.redis = RedisClient() self.redis.remove(CLIENT_NAME) print('Successfully Removed Proxy') except: print('nothing to remove') def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ self.redis = RedisClient() if self.redis.set(CLIENT_NAME, proxy): print('Successfully Set Proxy', proxy) def adsl(self): """ 拨号主进程 :return: None """ while True: print('ADSL Start, Remove Proxy, Please wait') self.remove_proxy() (status, output) = subprocess.getstatusoutput(ADSL_BASH) if status == 0: print('ADSL Successfully') ip = self.get_ip() if ip: print('Now IP', ip) print('Testing Proxy, Please Wait') proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT) if self.test_proxy(proxy): print('Valid Proxy') self.set_proxy(proxy) print('Sleeping') time.sleep(ADSL_CYCLE) else: print('Invalid Proxy') else: print('Get IP Failed, Re Dialing') time.sleep(ADSL_ERROR_CYCLE) else: print('ADSL Failed, Please Check') time.sleep(ADSL_ERROR_CYCLE)
# coding=utf-8 from adslproxy.api import server from adslproxy.db import RedisClient from adslproxy.config import * if __name__ == '__main__': redis = RedisClient(host=REDIS_HOST, password=REDIS_PASSWORD) server(redis=redis)
class Sender(object): """ 拨号并发送到 Redis """ ip_pre = '' invalid_ip_list = [] def extract_ip(self): """ 获取本机IP :param ifname: 网卡名称 :return: """ (status, output) = subprocess.getstatusoutput('ifconfig') if not status == 0: return pattern = re.compile(DIAL_IFNAME + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S) result = re.search(pattern, output) if result: # 返回拨号后的 IP 地址 return result.group(1) def test_proxy(self, proxy): """ 测试代理,返回测试结果 :param proxy: 代理 :return: 测试结果 :ping一次测试速度更快,只需要几十毫秒 """ # try: # response = requests.get(TEST_URL, proxies={ # 'http': 'http://' + proxy, # 'https': 'https://' + proxy # }, timeout=TEST_TIMEOUT) # if response.status_code == 200: # logger.info(f'proxy: {proxy}') # return True #except (ConnectionError, ReadTimeout): # return False con = os.system('ping -c 1 www.baidu.com') print(con) if con==0: return True else: return False @retry(retry_on_result=lambda x: x is not True, stop_max_attempt_number=10) def remove_proxy(self): """ 移除代理 :return: None 通常情况下,连续拨号失败几次就需要重启机器了,这时候VPS已经无法成功拨号连接互联网了 """ logger.info(f'Removing {CLIENT_NAME}...') try: # 由于拨号就会中断连接,所以每次都要重新建立连接 if hasattr(self, 'redis') and self.redis: self.redis.close() self.redis = RedisClient() self.redis.remove(CLIENT_NAME) logger.info(f'Removed {CLIENT_NAME} successfully') return True except redis.ConnectionError: logger.info(f'Remove {CLIENT_NAME} failed') logger.error('删除IP失败!从代理池删除IP并重启系统.......') os.system('/usr/sbin/shutdown -r now') def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ self.redis = RedisClient() self.db = RedisClient().db # 哈希表来统计拨号VPS的IP if not self.db.hexists('dialed_IPs', proxy): self.db.hset('dialed_IPs', proxy, 1) # 往IP池里插入数据 if self.redis.set(CLIENT_NAME, proxy): logger.info(f'Successfully set proxy {proxy}') return True else: num = int(self.db.hget('dialed_IPs', proxy)) logger.info(f'{proxy} in proxy pools {num} times already') if num <2: self.db.hset('dialed_IPs', proxy, num+1) # 往IP池里插入数据 if self.redis.set(CLIENT_NAME, proxy): logger.info(f'Successfully set proxy {proxy}') return True else: return False def loop(self): """ 循环拨号 :return: """ while True: logger.info('Starting dial...') now = datetime.datetime.now() if now.minute%5==0 and now.second==0: logger.info('dial time: %s', now.strftime('%Y-%m-%d %H:%M:%S')) new_ip = self.run() if new_ip != self.ip_pre: self.ip_pre = new_ip else: logger.info('IP和上次相同,等待重播......') self.run() def run(self): """ 拨号主进程 :return: None """ #time.sleep(10) #给正在运行的作业留出时间结束 logger.info('Dial started, remove proxy') try: self.remove_proxy() except RetryError: logger.error('Retried for max times, continue') self.emailclient = EmailClient() self.emailclient.notification(f'failed too many times {datetime.datetime.now().strftime("%m-%d-%H-%M")}', f'Warning{random.randint(1000,299999)}: 22457 retry error {datetime.datetime.now().strftime("%m-%d-%H-%M")}') for i in range(3): # 拨号 (status, output) = subprocess.getstatusoutput('adsl-stop;adsl-start') if not status == 0: logger.error('Dial failed') time.sleep(20) else: break if not status == 0: print('连续三次拨号失败,系统重启......') os.system('sudo reboot') # 获取拨号 IP ip = self.extract_ip() if ip: logger.info(f'Get new IP {ip}') if PROXY_USERNAME and PROXY_PASSWORD: proxy = '{username}:{password}@{ip}:{port}'.format(username=PROXY_USERNAME, password=PROXY_PASSWORD, ip=ip, port=PROXY_PORT) else: proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT) # time.sleep(1) if self.test_proxy(proxy): logger.info(f'Valid proxy {proxy}') self.ip_validity_statistics('valid') # 将代理放入数据库 if self.set_proxy(proxy): time.sleep(DIAL_CYCLE) else: logger.error(f'Proxy invalid {proxy}') # 连续三次拨号无效 self.ip_validity_statistics('invalid') if len(self.invalid_ip_list) > 0: if self.invalid_ip_list.count('invalid') == 3: logger.error('连续三次拨号失败!从代理池删除IP并重启系统.......') self.remove_proxy() os.system('/usr/sbin/shutdown -r now') time.sleep(DIAL_ERROR_CYCLE) else: # 获取 IP 失败,重新拨号 logger.error('Get IP failed, re-dialing') ip = '' time.sleep(DIAL_ERROR_CYCLE) self.run() return ip def ip_validity_statistics(self, ele): if len(self.invalid_ip_list) < 3: self.invalid_ip_list.append(ele) else: self.invalid_ip_list.pop(0) self.invalid_ip_list.append(ele)
# coding=utf-8 from adslproxy.api import server from adslproxy.db import RedisClient import os passwd = os.getenv('REDISPASSWORD') if __name__ == '__main__': redis = RedisClient(host='', password=passwd) server(redis=redis, port=8000)
def __init__(self): self.db = RedisClient() self.counts = defaultdict(int)
class Sender(): def get_ip(self, ifname=ADSL_IFNAME): """ 获取本机IP :param ifname: 网卡名称 :return: """ # attention ->程序自动化执行命令 (status, output) = subprocess.getstatusoutput('ifconfig') if status == 0: # attention ->对照所显示出的端口信息/ 使用regex做匹配获取ip pattern = re.compile( ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S) result = re.search(pattern, output) if result: ip = result.group(1) return ip def test_proxy(self, proxy): """ 测试代理 :param proxy: 代理 :return: 测试结果 """ try: # attention +-- response = requests.get(TEST_URL, proxies={ 'http': 'http://' + proxy, 'https': 'https://' + proxy }, timeout=TEST_TIMEOUT) if response.status_code == 200: return True except (ConnectionError, ReadTimeout): return False def remove_proxy(self): """ 移除代理 :return: None """ self.redis = RedisClient() # 删除对应主机 self.redis.remove(CLIENT_NAME) print('Successfully Removed Proxy') def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ self.redis = RedisClient() if self.redis.set(CLIENT_NAME, proxy): print('Successfully Set Proxy', proxy) def adsl(self): """ 拨号主进程 :return: None """ while True: print('ADSL Start, Remove Proxy, Please wait') # attention ---------------> try: # 拨号之前将已有的redis散列做清空 self.remove_proxy() except: while True: # 重启拨号 (status, output) = subprocess.getstatusoutput(ADSL_BASH) if status == 0: self.remove_proxy() break # -------------------------- (status, output) = subprocess.getstatusoutput(ADSL_BASH) if status == 0: print('ADSL Successfully') ip = self.get_ip() if ip: print('New IP', ip) print('Testing Proxy, Please Wait') proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT) if self.test_proxy(proxy): print('Valid Proxy') self.set_proxy(proxy) print('Sleeping') time.sleep(ADSL_CYCLE) else: print('Invalid Proxy') else: print('Get IP Failed, Re Dialing') time.sleep(ADSL_ERROR_CYCLE) else: print('ADSL Failed, Please Check') time.sleep(ADSL_ERROR_CYCLE)
class Sender(): def __init__(self): self.redis = RedisClient() # 日志打印格式 log_fmt = '%(asctime)s\tFile \"%(filename)s\",line %(lineno)s\t%(levelname)s: %(message)s' formatter = logging.Formatter(log_fmt) # 创建TimedRotatingFileHandler对象 self.log_file_handler = TimedRotatingFileHandler(filename="adsl.log", when="D", interval=1, backupCount=7) self.log_file_handler.suffix = "%Y-%m-%d_%H-%M.log" self.log_file_handler.extMatch = re.compile( r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}.log$") self.log_file_handler.setFormatter(formatter) logging.basicConfig(level=logging.INFO) self.log = logging.getLogger() self.log.addHandler(self.log_file_handler) def __del__(self): self.remove_proxy() def get_ip(self, ifname=ADSL_IFNAME): """ 获取本机IP :param ifname: 网卡名称 :return: """ (status, output) = subprocess.getstatusoutput('ifconfig') if status == 0: pattern = re.compile( ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S) result = re.search(pattern, output) if result: ip = result.group(1) return ip def test_proxy(self, proxy): """ 测试代理 :param proxy: 代理 :return: 测试结果 """ try: response = requests.get( TEST_URL, proxies={ 'http': 'http://' + proxy, #'https': 'https://' + proxy }, timeout=TEST_TIMEOUT) if response.status_code == 200: return True except (ConnectionError, ReadTimeout): return False def remove_proxy(self): """ 移除代理 :return: None """ # TODO # self.redis = RedisClient() self.redis.remove(CLIENT_NAME) self.log.info("Successfully Removed Proxy") self.log.addHandler(self.log_file_handler) def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ # TODO # self.redis = RedisClient() if self.redis.set(CLIENT_NAME, proxy): self.log.info("Successfully Set Proxy : %s" % proxy) self.log.addHandler(self.log_file_handler) def adsl(self): """ 拨号主进程 :return: None """ while True: self.log.info( "ADSL is Starting, And Removing Proxy, Please wait......") self.log.addHandler(self.log_file_handler) try: self.remove_proxy() self.redis = None except: while True: (status, output) = subprocess.getstatusoutput(ADSL_BASH) if status == 0: self.redis = RedisClient() self.remove_proxy() self.redis = None break self.redis = None (status, output) = subprocess.getstatusoutput(ADSL_BASH) self.redis = RedisClient() if status == 0: self.log.info("ADSL Successfully") self.log.addHandler(self.log_file_handler) ip = self.get_ip() if ip: self.log.info("The new IP is : %s" % ip) self.log.info("Testing Proxy, Please Wait") self.log.addHandler(self.log_file_handler) proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT) if self.test_proxy(proxy): self.log.info("This is a valid proxy") self.set_proxy(proxy) self.log.info('The program is sleeping for %s second' % ADSL_CYCLE) self.log.addHandler(self.log_file_handler) time.sleep(ADSL_CYCLE) else: self.log.warning("Oops.This is a invalid proxy") self.log.addHandler(self.log_file_handler) else: self.log.warning("Get IP Failed, Re Dialing") self.log.addHandler(self.log_file_handler) time.sleep(ADSL_ERROR_CYCLE) else: self.log.warning("Oops.ADSL Failed, Please Check") self.log.addHandler(self.log_file_handler) time.sleep(ADSL_ERROR_CYCLE)
class Sender(): def get_ip(self, ifname=ADSL_IFNAME): """ 获取本机IP :param ifname: 网卡名称 :return: """ (status, output) = subprocess.getstatusoutput('ifconfig') if status == 0: pattern = re.compile(ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S) result = re.search(pattern, output) if result: ip = result.group(1) return ip def test_proxy(self, proxy): """ 测试代理 :param proxy: 代理 :return: 测试结果 """ try: response = requests.get(TEST_URL, proxies={ 'http': 'http://' + proxy, 'https': 'https://' + proxy }, timeout=TEST_TIMEOUT) if response.status_code == 200: return True except (ConnectionError, ReadTimeout): return False def remove_proxy(self): """ 移除代理 :return: None """ try: self.redis = RedisClient() self.redis.remove(CLIENT_NAME) except: print('remove proxy failed') print('Successfully Removed Proxy') def set_proxy(self, proxy): """ 设置代理 :param proxy: 代理 :return: None """ self.redis = RedisClient() if self.redis.set(CLIENT_NAME, proxy): print('Successfully Set Proxy', proxy) def adsl(self): """ 拨号主进程 :return: None """ while True: print('ADSL Start, Remove Proxy, Please wait') self.remove_proxy() # (status, output) = subprocess.getstatusoutput(ADSL_BASH) subprocess.getstatusoutput("pppoe-stop") time.sleep(4) subprocess.getstatusoutput("/bin/systemctl stop NetworkManager.service") time.sleep(3) (status, output) = subprocess.getstatusoutput("pppoe-start") if status == 0: print('ADSL Successfully') ip = self.get_ip() if ip: print('Now IP', ip) print('Testing Proxy, Please Wait') proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT) if self.test_proxy(proxy): print('Valid Proxy') self.set_proxy(proxy) print('Sleeping') time.sleep(ADSL_CYCLE) else: print('Invalid Proxy') else: print('Get IP Failed, Re Dialing') time.sleep(ADSL_ERROR_CYCLE) else: print('ADSL Failed, Please Check') time.sleep(ADSL_ERROR_CYCLE)