class Sender():
def get_ip(self, ifname=ADSL_IFNAME):
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
pattern = re.compile(
ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
try:
response = requests.get(TEST_URL,
proxies={
'http': 'http://' + proxy,
'https': 'https://' + proxy
},
timeout=TEST_TIMEOUT)
if response.status_code == 200:
return True
except (ConnectionError, ReadTimeout):
return False
def remove_proxy(self):
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
def set_proxy(self, proxy):
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
def adsl(self):
while True:
print('ADSL Start, Remove Proxy, Please wait')
self.remove_proxy()
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
print('ADSL Successfully')
ip = self.get_ip()
if ip:
print('Now IP', ip)
print('Testing Proxy, Please Wait')
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
if self.test_proxy(proxy):
print('Valid Proxy')
self.set_proxy(proxy)
print('Sleeping')
time.sleep(ADSL_CYCLE)
else:
print('Invalid Proxy')
else:
print('Get IP Failed, Re Dialing')
time.sleep(ADSL_ERROR_CYCLE)
else:
print('ADSL Failed, Please Check')
time.sleep(ADSL_ERROR_CYCLE)
class Checker(object):
def __init__(self):
self.db = RedisClient()
self.counts = defaultdict(int)
def check(self, proxy):
"""
测试代理，返回测试结果
:param proxy: 代理
:return: 测试结果
"""
try:
response = requests.get(settings.TEST_URL,
proxies={
'http': 'http://' + proxy,
'https': 'https://' + proxy
},
timeout=settings.TEST_TIMEOUT)
logger.debug(f'Using {proxy} to test {settings.TEST_URL}...')
if response.status_code == 200:
return True
except (ConnectionError, ReadTimeout):
return False
def run(self):
"""
测试一轮
:return:
"""
proxies = self.db.all()
logger.info(f'Try to get all proxies {proxies}')
for name, proxy in proxies.items():
# 检测无效
if not self.check(proxy):
logger.info(f'Proxy {proxy} invalid')
self.counts[proxy] += 1
else:
logger.info(f'Proxy {proxy} valid')
count = self.counts.get(proxy) or 0
logger.debug(
f'Count {count}, TEST_MAX_ERROR_COUNT {settings.TEST_MAX_ERROR_COUNT}'
)
if count >= settings.TEST_MAX_ERROR_COUNT:
self.db.remove(name)
def loop(self):
"""
循环测试
:return:
"""
while True:
logger.info('Check for infinite')
self.run()
logger.info(f'Tested, sleeping for {settings.TEST_CYCLE}s...')
time.sleep(settings.TEST_CYCLE)
class Sender():
def get_ip(self, ifname=ADSL_IFNAME):
"""
获取本机IP
:param ifname: 网卡名称
:return:
"""
# attention ->程序自动化执行命令
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
# attention ->对照所显示出的端口信息/ 使用regex做匹配获取ip
pattern = re.compile(
ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
"""
测试代理
:param proxy: 代理
:return: 测试结果
"""
try:
# attention +--
response = requests.get(TEST_URL,
proxies={
'http': 'http://' + proxy,
'https': 'https://' + proxy
},
timeout=TEST_TIMEOUT)
if response.status_code == 200:
return True
except (ConnectionError, ReadTimeout):
return False
def remove_proxy(self):
"""
移除代理
:return: None
"""
self.redis = RedisClient()
# 删除对应主机
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
def adsl(self):
"""
拨号主进程
:return: None
"""
while True:
print('ADSL Start, Remove Proxy, Please wait')
# attention --------------->
try:
# 拨号之前将已有的redis散列做清空
self.remove_proxy()
except:
while True:
# 重启拨号
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
self.remove_proxy()
break
# --------------------------
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
print('ADSL Successfully')
ip = self.get_ip()
if ip:
print('New IP', ip)
print('Testing Proxy, Please Wait')
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
if self.test_proxy(proxy):
print('Valid Proxy')
self.set_proxy(proxy)
print('Sleeping')
time.sleep(ADSL_CYCLE)
else:
print('Invalid Proxy')
else:
print('Get IP Failed, Re Dialing')
time.sleep(ADSL_ERROR_CYCLE)
else:
print('ADSL Failed, Please Check')
time.sleep(ADSL_ERROR_CYCLE)
class Sender(object):
"""
拨号并发送到 Redis
"""
def extract_ip(self):
"""
获取本机IP
:param ifname: 网卡名称
:return:
"""
(status, output) = subprocess.getstatusoutput('ifconfig')
if not status == 0: return
pattern = re.compile(DIAL_IFNAME + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
# 返回拨号后的 IP 地址
return result.group(1)
def test_proxy(self, proxy):
"""
测试代理，返回测试结果
:param proxy: 代理
:return: 测试结果
"""
try:
response = requests.get(TEST_URL, proxies={
'http': 'http://' + proxy,
'https': 'https://' + proxy
}, timeout=TEST_TIMEOUT)
if response.status_code == 200:
return True
except (ConnectionError, ReadTimeout):
return False
@retry(retry_on_result=lambda x: x is not True, stop_max_attempt_number=10)
def remove_proxy(self):
"""
移除代理
:return: None
"""
logger.info(f'Removing {CLIENT_NAME}...')
try:
# 由于拨号就会中断连接，所以每次都要重新建立连接
if hasattr(self, 'redis') and self.redis:
self.redis.close()
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
logger.info(f'Removed {CLIENT_NAME} successfully')
return True
except redis.ConnectionError:
logger.info(f'Remove {CLIENT_NAME} failed')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
logger.info(f'Successfully set proxy {proxy}')
def loop(self):
"""
循环拨号
:return:
"""
while True:
logger.info('Starting dial...')
self.run()
time.sleep(DIAL_CYCLE)
def run(self):
"""
拨号主进程
:return: None
"""
logger.info('Dial started, remove proxy')
try:
self.remove_proxy()
except RetryError:
logger.error('Retried for max times, continue')
# 拨号
(status, output) = subprocess.getstatusoutput(DIAL_BASH)
if not status == 0:
logger.error('Dial failed')
# 获取拨号 IP
ip = self.extract_ip()
if ip:
logger.info(f'Get new IP {ip}')
if PROXY_USERNAME and PROXY_PASSWORD:
proxy = '{username}:{password}@{ip}:{port}'.format(username=PROXY_USERNAME,
password=PROXY_PASSWORD,
ip=ip, port=PROXY_PORT)
else:
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
time.sleep(10)
if self.test_proxy(proxy):
logger.info(f'Valid proxy {proxy}')
# 将代理放入数据库
self.set_proxy(proxy)
time.sleep(DIAL_CYCLE)
else:
logger.error(f'Proxy invalid {proxy}')
else:
# 获取 IP 失败，重新拨号
logger.error('Get IP failed, re-dialing')
self.run()
class Sender():
def get_ip(self, ifname=ADSL_IFNAME):
"""
获取本机IP
:param ifname: 网卡名称
:return:
"""
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
pattern = re.compile(
ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
"""
测试代理
:param proxy: 代理
:return: 测试结果
"""
headers = {
'Accept':
'text/html, application/xhtml+xml, image/jxr, */*',
'Accept - Encoding':
'gzip, deflate',
'Accept-Language':
'zh-Hans-CN, zh-Hans; q=0.5',
'Connection':
'Keep-Alive',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063'
}
try:
response = requests.get(TEST_URL,
headers=headers,
proxies={
'http': 'http://' + proxy,
'https': 'https://' + proxy
},
timeout=TEST_TIMEOUT)
if response.status_code == 200:
return True
else:
print('response_code is', response.status_code)
return False
except (ConnectionError, ReadTimeout):
return False
def remove_proxy(self):
"""
移除代理
:return: None
"""
try:
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
except:
print('nothing to remove')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
def adsl(self):
"""
拨号主进程
:return: None
"""
while True:
print('ADSL Start, Remove Proxy, Please wait')
self.remove_proxy()
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
print('ADSL Successfully')
ip = self.get_ip()
if ip:
print('Now IP', ip)
print('Testing Proxy, Please Wait')
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
if self.test_proxy(proxy):
print('Valid Proxy')
self.set_proxy(proxy)
print('Sleeping')
time.sleep(ADSL_CYCLE)
else:
print('Invalid Proxy')
else:
print('Get IP Failed, Re Dialing')
time.sleep(ADSL_ERROR_CYCLE)
else:
print('ADSL Failed, Please Check')
time.sleep(ADSL_ERROR_CYCLE)
class Sender(object):
"""
拨号并发送到 Redis
"""
ip_pre = ''
invalid_ip_list = []
def extract_ip(self):
"""
获取本机IP
:param ifname: 网卡名称
:return:
"""
(status, output) = subprocess.getstatusoutput('ifconfig')
if not status == 0: return
pattern = re.compile(DIAL_IFNAME + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
# 返回拨号后的 IP 地址
return result.group(1)
def test_proxy(self, proxy):
"""
测试代理，返回测试结果
:param proxy: 代理
:return: 测试结果
:ping一次测试速度更快，只需要几十毫秒
"""
# try:
# response = requests.get(TEST_URL, proxies={
# 'http': 'http://' + proxy,
# 'https': 'https://' + proxy
# }, timeout=TEST_TIMEOUT)
# if response.status_code == 200:
# logger.info(f'proxy: {proxy}')
# return True
#except (ConnectionError, ReadTimeout):
# return False
con = os.system('ping -c 1 www.baidu.com')
print(con)
if con==0:
return True
else:
return False
@retry(retry_on_result=lambda x: x is not True, stop_max_attempt_number=10)
def remove_proxy(self):
"""
移除代理
:return: None
通常情况下，连续拨号失败几次就需要重启机器了，这时候VPS已经无法成功拨号连接互联网了
"""
logger.info(f'Removing {CLIENT_NAME}...')
try:
# 由于拨号就会中断连接，所以每次都要重新建立连接
if hasattr(self, 'redis') and self.redis:
self.redis.close()
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
logger.info(f'Removed {CLIENT_NAME} successfully')
return True
except redis.ConnectionError:
logger.info(f'Remove {CLIENT_NAME} failed')
logger.error('删除IP失败!从代理池删除IP并重启系统.......')
os.system('/usr/sbin/shutdown -r now')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
self.db = RedisClient().db
# 哈希表来统计拨号VPS的IP
if not self.db.hexists('dialed_IPs', proxy):
self.db.hset('dialed_IPs', proxy, 1)
# 往IP池里插入数据
if self.redis.set(CLIENT_NAME, proxy):
logger.info(f'Successfully set proxy {proxy}')
return True
else:
num = int(self.db.hget('dialed_IPs', proxy))
logger.info(f'{proxy} in proxy pools {num} times already')
if num <2:
self.db.hset('dialed_IPs', proxy, num+1)
# 往IP池里插入数据
if self.redis.set(CLIENT_NAME, proxy):
logger.info(f'Successfully set proxy {proxy}')
return True
else:
return False
def loop(self):
"""
循环拨号
:return:
"""
while True:
logger.info('Starting dial...')
now = datetime.datetime.now()
if now.minute%5==0 and now.second==0:
logger.info('dial time: %s', now.strftime('%Y-%m-%d %H:%M:%S'))
new_ip = self.run()
if new_ip != self.ip_pre:
self.ip_pre = new_ip
else:
logger.info('IP和上次相同，等待重播......')
self.run()
def run(self):
"""
拨号主进程
:return: None
"""
#time.sleep(10) #给正在运行的作业留出时间结束
logger.info('Dial started, remove proxy')
try:
self.remove_proxy()
except RetryError:
logger.error('Retried for max times, continue')
self.emailclient = EmailClient()
self.emailclient.notification(f'failed too many times {datetime.datetime.now().strftime("%m-%d-%H-%M")}', f'Warning{random.randint(1000,299999)}: 22457 retry error {datetime.datetime.now().strftime("%m-%d-%H-%M")}')
for i in range(3):
# 拨号
(status, output) = subprocess.getstatusoutput('adsl-stop;adsl-start')
if not status == 0:
logger.error('Dial failed')
time.sleep(20)
else:
break
if not status == 0:
print('连续三次拨号失败,系统重启......')
os.system('sudo reboot')
# 获取拨号 IP
ip = self.extract_ip()
if ip:
logger.info(f'Get new IP {ip}')
if PROXY_USERNAME and PROXY_PASSWORD:
proxy = '{username}:{password}@{ip}:{port}'.format(username=PROXY_USERNAME,
password=PROXY_PASSWORD,
ip=ip, port=PROXY_PORT)
else:
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
# time.sleep(1)
if self.test_proxy(proxy):
logger.info(f'Valid proxy {proxy}')
self.ip_validity_statistics('valid')
# 将代理放入数据库
if self.set_proxy(proxy):
time.sleep(DIAL_CYCLE)
else:
logger.error(f'Proxy invalid {proxy}')
# 连续三次拨号无效
self.ip_validity_statistics('invalid')
if len(self.invalid_ip_list) > 0:
if self.invalid_ip_list.count('invalid') == 3:
logger.error('连续三次拨号失败!从代理池删除IP并重启系统.......')
self.remove_proxy()
os.system('/usr/sbin/shutdown -r now')
time.sleep(DIAL_ERROR_CYCLE)
else:
# 获取 IP 失败，重新拨号
logger.error('Get IP failed, re-dialing')
ip = ''
time.sleep(DIAL_ERROR_CYCLE)
self.run()
return ip
def ip_validity_statistics(self, ele):
if len(self.invalid_ip_list) < 3:
self.invalid_ip_list.append(ele)
else:
self.invalid_ip_list.pop(0)
self.invalid_ip_list.append(ele)
class Sender():
def __init__(self):
self.redis = RedisClient()
# 日志打印格式
log_fmt = '%(asctime)s\tFile \"%(filename)s\",line %(lineno)s\t%(levelname)s: %(message)s'
formatter = logging.Formatter(log_fmt)
# 创建TimedRotatingFileHandler对象
self.log_file_handler = TimedRotatingFileHandler(filename="adsl.log",
when="D",
interval=1,
backupCount=7)
self.log_file_handler.suffix = "%Y-%m-%d_%H-%M.log"
self.log_file_handler.extMatch = re.compile(
r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}.log$")
self.log_file_handler.setFormatter(formatter)
logging.basicConfig(level=logging.INFO)
self.log = logging.getLogger()
self.log.addHandler(self.log_file_handler)
def __del__(self):
self.remove_proxy()
def get_ip(self, ifname=ADSL_IFNAME):
"""
获取本机IP
:param ifname: 网卡名称
:return:
"""
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
pattern = re.compile(
ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
"""
测试代理
:param proxy: 代理
:return: 测试结果
"""
try:
response = requests.get(
TEST_URL,
proxies={
'http': 'http://' + proxy,
#'https': 'https://' + proxy
},
timeout=TEST_TIMEOUT)
if response.status_code == 200:
return True
except (ConnectionError, ReadTimeout):
return False
def remove_proxy(self):
"""
移除代理
:return: None
"""
# TODO
# self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
self.log.info("Successfully Removed Proxy")
self.log.addHandler(self.log_file_handler)
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
# TODO
# self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
self.log.info("Successfully Set Proxy : %s" % proxy)
self.log.addHandler(self.log_file_handler)
def adsl(self):
"""
拨号主进程
:return: None
"""
while True:
self.log.info(
"ADSL is Starting, And Removing Proxy, Please wait......")
self.log.addHandler(self.log_file_handler)
try:
self.remove_proxy()
self.redis = None
except:
while True:
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
self.redis = RedisClient()
self.remove_proxy()
self.redis = None
break
self.redis = None
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
self.redis = RedisClient()
if status == 0:
self.log.info("ADSL Successfully")
self.log.addHandler(self.log_file_handler)
ip = self.get_ip()
if ip:
self.log.info("The new IP is : %s" % ip)
self.log.info("Testing Proxy, Please Wait")
self.log.addHandler(self.log_file_handler)
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
if self.test_proxy(proxy):
self.log.info("This is a valid proxy")
self.set_proxy(proxy)
self.log.info('The program is sleeping for %s second' %
ADSL_CYCLE)
self.log.addHandler(self.log_file_handler)
time.sleep(ADSL_CYCLE)
else:
self.log.warning("Oops.This is a invalid proxy")
self.log.addHandler(self.log_file_handler)
else:
self.log.warning("Get IP Failed, Re Dialing")
self.log.addHandler(self.log_file_handler)
time.sleep(ADSL_ERROR_CYCLE)
else:
self.log.warning("Oops.ADSL Failed, Please Check")
self.log.addHandler(self.log_file_handler)
time.sleep(ADSL_ERROR_CYCLE)
class Sender():
def get_ip(self, ifname=ADSL_IFNAME):
"""
获取本机IP
:param ifname: 网卡名称
:return:
"""
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
pattern = re.compile(ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
"""
测试代理
:param proxy: 代理
:return: 测试结果
"""
try:
response = requests.get(TEST_URL, proxies={
'http': 'http://' + proxy,
'https': 'https://' + proxy
}, timeout=TEST_TIMEOUT)
if response.status_code == 200:
return True
except (ConnectionError, ReadTimeout):
return False
def remove_proxy(self):
"""
移除代理
:return: None
"""
try:
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
except:
print('remove proxy failed')
print('Successfully Removed Proxy')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
def adsl(self):
"""
拨号主进程
:return: None
"""
while True:
print('ADSL Start, Remove Proxy, Please wait')
self.remove_proxy()
# (status, output) = subprocess.getstatusoutput(ADSL_BASH)
subprocess.getstatusoutput("pppoe-stop")
time.sleep(4)
subprocess.getstatusoutput("/bin/systemctl stop NetworkManager.service")
time.sleep(3)
(status, output) = subprocess.getstatusoutput("pppoe-start")
if status == 0:
print('ADSL Successfully')
ip = self.get_ip()
if ip:
print('Now IP', ip)
print('Testing Proxy, Please Wait')
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
if self.test_proxy(proxy):
print('Valid Proxy')
self.set_proxy(proxy)
print('Sleeping')
time.sleep(ADSL_CYCLE)
else:
print('Invalid Proxy')
else:
print('Get IP Failed, Re Dialing')
time.sleep(ADSL_ERROR_CYCLE)
else:
print('ADSL Failed, Please Check')
time.sleep(ADSL_ERROR_CYCLE)
class Sender():
def get_ip(self, ifname=ADSL_IFNAME):
"""
获取本机IP
:param ifname: 网卡名称
:return:
"""
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
pattern = re.compile(ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
"""
测试代理
:param proxy: 代理
:return: 测试结果
"""
try:
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}
response = requests.get(TEST_URL, headers=headers, proxies={
'http': 'http://' + proxy,
'https': 'https://' + proxy
}, timeout=TEST_TIMEOUT)
if response.status_code == 200:
return True
except (ConnectionError, ReadTimeout):
return False
def remove_proxy(self):
"""
移除代理
:return: None
"""
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
def adsl(self):
"""
拨号主进程
:return: None
"""
while True:
print('ADSL Start, Remove Proxy, Please wait')
try:
self.remove_proxy()
except:
while True:
# (status, output) = subprocess.getstatusoutput(ADSL_BASH)
(status, output) = subprocess.getstatusoutput(ADSL_BASH_STOP)
if status == 0:
print("adsl-start after {} seconds".format(ADSL_STEP))
time.sleep(ADSL_STEP)
(status, output) = subprocess.getstatusoutput(ADSL_BASH_START)
if status == 0:
self.remove_proxy()
break
# adsl-stop;adsl-start间隔太短会造成IP不能改变
# (status, output) = subprocess.getstatusoutput(ADSL_BASH)
(status, output) = subprocess.getstatusoutput(ADSL_BASH_STOP)
if status == 0:
print("adsl-start after {} seconds".format(ADSL_STEP))
time.sleep(ADSL_STEP)
(status, output) = subprocess.getstatusoutput(ADSL_BASH_START)
if status == 0:
print('ADSL Successfully')
ip = self.get_ip()
if ip:
print('Now IP', ip)
print('Testing Proxy, Please Wait')
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
if self.test_proxy(proxy):
print('Valid Proxy')
self.set_proxy(proxy)
print('Sleeping')
pbar = tqdm(total=ADSL_CYCLE)
for i in range(int(ADSL_CYCLE/2)):
time.sleep(2)
pbar.update(2)
pbar.close()
# time.sleep(ADSL_CYCLE)
else:
print('Invalid Proxy')
else:
print('Get IP Failed, Re Dialing')
time.sleep(ADSL_ERROR_CYCLE)
else:
print('ADSL Failed, Please Check')
time.sleep(ADSL_ERROR_CYCLE)
class Sender():
def get_ip(self, ifname=ADSL_IFNAME):
"""
获取本机IP
:param ifname: 网卡名称
:return:
"""
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
pattern = re.compile(
ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
"""
测试代理
:param proxy: 代理
:return: 测试结果
"""
try:
response = requests.get(
TEST_URL,
#proxies={
# 'http': 'http://' + proxy,
# 'https': 'https://' + proxy
# },
#timeout=TEST_TIMEOUT
)
if response.status_code < 400:
return True
except (ConnectionError, ReadTimeout):
return False
def remove_proxy(self):
"""
移除代理
:return: None
"""
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
def write_conf(self, ip):
f = open(FILE_NAME, 'r+')
lines = f.readlines()
lines[11] = ' server_name {};\n'.format(ip)
f = open(FILE_NAME, 'w+')
f.writelines(lines)
f.close()
def adsl(self):
"""
拨号主进程
:return: None
"""
while True:
print('ADSL Start, Remove Proxy, Please wait')
self.remove_proxy()
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
print('ADSL Successfully')
ip = self.get_ip()
if ip:
print('Now IP', ip)
print('Testing Proxy, Please Wait')
proxy = '{ip}'.format(ip=ip)
self.set_proxy(proxy)
self.write_conf(ip)
(status, output) = subprocess.getstatusoutput(CONF_BASH)
if status == 0:
print('CONF Changed and Reloaded!')
else:
continue
if self.test_proxy(proxy):
print('Valid Proxy')
#self.set_proxy(proxy)
print('Sleeping')
time.sleep(ADSL_CYCLE)
else:
print('Invalid Proxy')
else:
print('Get IP Failed, Re Dialing')
time.sleep(ADSL_ERROR_CYCLE)
else:
print('ADSL Failed, Please Check')
time.sleep(ADSL_ERROR_CYCLE)
class Sender(object):
"""
拨号并发送到 Redis
"""
def __init__(self, server_ip, port, user, pwd, clent_name):
self.CLIENT_NAME = clent_name
self.ADSLHOST = server_ip
self.ADSLPORT = port
self.ADSLUSER = user
self.ADSLPWD = pwd
def test_proxy(self, proxy):
"""
测试代理，返回测试结果
:param proxy: 代理
:return: 测试结果
"""
try:
response = requests.get(
TEST_URL,
proxies={
'http': 'http://' + proxy,
'https': 'https://' + proxy,
# 'http': proxy,
# 'https': proxy
},
timeout=settings.TEST_TIMEOUT)
if response.status_code == 200:
return True
else:
return False
except (ConnectionError, ReadTimeout):
return False
def parseIfconfig(self, data):
data = data.split('\n\n')
data = [i for i in data if i and i.startswith('ppp0')]
ip = ""
for line in data:
re_ipaddr = re.compile(r'inet ([\d\.]{7,15})', re.M)
ip = re_ipaddr.search(line)
if ip:
ip = ip.group(1)
else:
ip = ''
return ip
@retry(retry_on_result=lambda x: x is not True, stop_max_attempt_number=10)
def remove_proxy(self):
"""
移除代理
:return: None
"""
logger.info(f'Removing {self.CLIENT_NAME}...')
try:
# 由于拨号就会中断连接，所以每次都要重新建立连接
if hasattr(self, 'redis') and self.redis:
self.redis.close()
self.redis = RedisClient()
self.redis.remove(self.CLIENT_NAME)
logger.info(f'Removed {self.CLIENT_NAME} successfully')
return True
except redis.ConnectionError:
logger.info(f'Remove {self.CLIENT_NAME} failed')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(self.CLIENT_NAME, proxy):
logger.info(f'Successfully set proxy {proxy}')
def loop(self):
"""
循环拨号
:return:
"""
while True:
logger.info('Starting dial...')
self.run()
time.sleep(settings.DIAL_CYCLE)
def run(self):
"""
拨号主进程
:return: None
"""
logger.info('Dial started, remove proxy')
try:
self.remove_proxy()
except RetryError:
logger.error('Retried for max times, continue')
# 拨号
m = Monitor(self.ADSLHOST, self.ADSLPORT, self.ADSLUSER, self.ADSLPWD)
m.link_server("adsl-stop")
m.link_server("adsl-start")
content = m.link_server("ifconfig")
m.close_net()
ip = self.parseIfconfig(content)
if ip:
proxy = "%s:%s" % (ip, settings.PROXY_PORT)
if self.test_proxy(proxy):
logger.info(f'Valid proxy {proxy}')
# 将代理放入数据库
self.set_proxy(proxy)
time.sleep(2)
return True
else:
logger.error(f'Proxy invalid {proxy}')
return False
else:
# 获取 IP 失败，重新拨号
logger.error('Get IP failed, re-dialing')
self.run()
class Sender():
def get_ip(self, ifname=ADSL_IFNAME):
"""
获取本机IP
:param ifname: 网卡名称
:return:
"""
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
pattern = re.compile(ifname + '.*?inet addr:(\d+\.\d+\.\d+\.\d+)',
re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
"""
测试代理
:param proxy: 代理
:return: 测试结果
"""
try:
response = requests.get(TEST_URL,
proxies={
'http': proxy,
'https': proxy
},
timeout=TEST_TIMEOUT)
if response.status_code == 200:
return True
except (ConnectionError, ReadTimeout):
return False
def remove_proxy(self):
"""
移除代理
:return: None
"""
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
def adsl(self):
"""
拨号主进程
:return: None
"""
last_ip = None
while True:
print('ADSL Start, Remove Proxy, Please wait')
try:
self.remove_proxy()
except:
while True:
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0 and 'Connected!' in output:
self.remove_proxy()
break
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0 and 'Connected!' in output:
print('ADSL Successfully')
time.sleep(1)
ip = self.get_ip()
if ip and ip != last_ip:
print('Now IP', ip)
print('Testing Proxy, Please Wait')
last_ip = copy.copy(ip)
proxy = 'http://{username}:{password}@{ip}:{port}'.format(
username=PROXY_USERNAME,
password=PROXY_PASSWORD,
ip=ip,
port=PROXY_PORT)
if self.test_proxy(proxy):
print('Valid Proxy')
self.set_proxy(proxy)
print('Sleeping')
time.sleep(ADSL_CYCLE)
else:
print('Invalid Proxy')
else:
print('Get IP Failed Or Repeated, Re Dialing')
time.sleep(ADSL_ERROR_CYCLE)
else:
print('ADSL Failed, Please Check')
time.sleep(ADSL_ERROR_CYCLE)
class Sender(object):
def get_ip(self, ifname=ADSL_IFNAME):
"""
获取本机IP
:param ifname:网卡名称
:return:ip
"""
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
# 加入re.S修饰符，使.匹配包括换行符在内的所有字符
pattern = re.compile(
ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
"""
测试代理
:param proxy:代理
:return:测试结果
"""
proxies = {'http': 'http://' + proxy, 'https': 'https://' + proxy}
try:
response = requests.get(TEST_URL,
proxies=proxies,
timeout=TEST_TIMEOUT)
if response.status_code == 200:
return True
except (ConnectionError, ReadTimeout):
return False
def remove_proxy(self):
"""
移除代理
:return:None
"""
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
def set_proxy(self, proxy):
"""
设置代理
:param proxy:代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
def adsl(self):
"""
拨号主进程
:return:None
"""
while True:
print('ADSL Start, Remove Proxy, Please wait')
# 将远程Redis散列表中的本机对应的代理移除，避免拨号时本主机的残留代理被取到
try:
self.remove_proxy()
except:
while True:
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
self.remove_proxy()
break
# 拨号脚本：stop之后再start
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
print('ADSL Successfully')
ip = self.get_ip()
if ip:
print('Now IP', ip)
print('Testing Proxy,Please wait')
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
print('proxy info:', proxy)
if self.test_proxy(proxy):
print('Valid Proxy')
# 代理有效时，更新Redis散列表
self.set_proxy(proxy)
time.sleep(ADSL_CYCLE)
else:
print('Invaild Proxy')
else:
print('Get Ip Failed,Please Check')
time.sleep(ADSL_CYCLE)
else:
print('ADSL Failed,Please Check')
time.sleep(ADSL_CYCLE)
class Sender():
def get_ip(self, ifname=ADSL_IFNAME):
"""
获取本机IP
:param ifname: 网卡名称
:return:
"""
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
pattern = re.compile(
ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
"""
测试代理
:param proxy: 代理
:return: 测试结果
"""
try:
response = requests.get(TEST_URL,
proxies={
'http': 'http://' + proxy,
'https': 'https://' + proxy
},
timeout=TEST_TIMEOUT)
if response.status_code == 200:
return True
except (ConnectionError, ReadTimeout):
return False
def remove_proxy(self):
"""
移除代理
:return: None
"""
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('已移除该代理IP')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('已设置代理IP', proxy)
def adsl(self):
"""
拨号主进程
:return: None
"""
while True:
print('移除代理并开始ADSL拨号，请稍等...')
self.remove_proxy()
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
print('拨号成功')
ip = self.get_ip()
if ip:
print('现在的IP是', ip)
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
if self.test_proxy(proxy):
print('代理IP{}可用'.format(proxy))
self.set_proxy(proxy)
print('已将代理IP更新到Redis，将在{}秒后重新拨号'.format(ADSL_CYCLE))
time.sleep(ADSL_CYCLE)
else:
print('代理IP{}不可用'.format(proxy))
else:
print('获取IP失败，将在{}秒后重新拨号'.format(ADSL_CYCLE))
time.sleep(ADSL_ERROR_CYCLE)
else:
print('拨号失败，将在{}秒后重新拨号'.format(ADSL_CYCLE))
time.sleep(ADSL_ERROR_CYCLE)
class Sender():
async def get_test_url_response(self, proxy):
try:
print(TEST_URL, TEST_TIMEOUT, 'output config, debugging. 22----')
response = requests.get(TEST_URL,
proxies={
'http': 'http://' + proxy,
'https': 'https://' + proxy
},
timeout=TEST_TIMEOUT)
return response
except (ConnectionError, ReadTimeout):
return False
async def test_proxy(self, proxy):
"""
测试代理
:param proxy: 代理
:return: 测试结果
"""
response = await self.get_test_url_response(proxy)
print('33')
if response:
return 200 == response.status_code
else:
return False
# try:
# print(TEST_URL, TEST_TIMEOUT, 'output config, debugging. 22----')
# response = requests.get(TEST_URL, proxies={
# 'http': 'http://' + proxy,
# 'https': 'https://' + proxy
# }, timeout=TEST_TIMEOUT)
# if response.status_code == 200:
# return True
# except (ConnectionError, ReadTimeout):
# return False
def remove_proxy(self):
"""
移除代理
:return: None
"""
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
def set_proxy(self, proxy):
"""
设置代理
:param proxy: 代理
:return: None
"""
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
async def adsl(self):
"""
拨号主进程
:return: None
"""
while True:
print('ADSL Start, Remove Proxy, Please wait')
self.remove_proxy()
(status, output) = sureprocess.getstatusoutput(ADSL_BASH)
if status == 0:
print('ADSL Successfully')
try:
print(
'Starting to wait for the pppoe redail done, sleep 3s...'
)
time.sleep(3)
ip = await getoutip()
print('in adls method, ip has been got: ' + ip)
if ip:
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
# proxy = '183.230.145.112:8888'
proxy = '61.189.242.243:55484'
print('Now proxy, ' + proxy +
', testing Proxy, Please Wait')
ifvalid = await self.test_proxy(proxy)
if ifvalid:
print('Valid Proxy')
self.set_proxy(proxy)
print('Sleeping... ' + str(ADSL_CYCLE))
time.sleep(ADSL_CYCLE)
else:
print('Invalid Proxy')
except StopIteration:
print('Get IP Failed, Re Dialing')
print('Sleeping... ' + str(ADSL_CYCLE))
time.sleep(ADSL_ERROR_CYCLE)
else:
print('ADSL Failed, Please Check')
print('Sleeping... ' + str(ADSL_CYCLE))
time.sleep(ADSL_ERROR_CYCLE)
class Sender():
def __init__(self):
self.proxy = []
self.proxies = None
self.headers = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
]
def get_ip(self, ifname=ADSL_IFNAME):
(status, output) = subprocess.getstatusoutput('ifconfig')
if status == 0:
pattern = re.compile(
ifname + '.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask', re.S)
result = re.search(pattern, output)
if result:
ip = result.group(1)
return ip
def test_proxy(self, proxy):
try:
if proxy not in self.proxy:
print('start texting')
m = choice(self.headers)
headers = {'User-Agent': m}
html = rq.get(TEST_URL, headers=headers, timeout=5)
if html.status_code == 200:
self.proxy.append(proxy)
self.proxies = proxy + "\t" + m
if len(self.proxy) > 50:
self.proxy.remove(self.proxy[0])
#if '66.0.3359' in m:
#subprocess.getstatusoutput('systemctl restart tinyproxy.service')
return True
else:
self.proxy.append(proxy)
print("连接状态不是200")
if len(self.proxy) > 50:
self.proxy.remove(self.proxy[0])
return False
else:
print("和之前IP重复")
return False
except:
return False
def remove_proxy(self):
self.redis = RedisClient()
self.redis.remove(CLIENT_NAME)
print('Successfully Removed Proxy')
def set_proxy(self, proxy):
self.redis = RedisClient()
if self.redis.set(CLIENT_NAME, proxy):
print('Successfully Set Proxy', proxy)
def adsl(self):
while True:
print('ADSL Start, Please wait')
(status, output) = subprocess.getstatusoutput(ADSL_BASH)
if status == 0:
print('ADSL Successfully')
ip = self.get_ip()
if ip:
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
print("new proxy ", proxy)
if self.test_proxy(proxy):
print('Valid Proxy')
self.set_proxy(self.proxies)
print('Sleeping', ADSL_CYCLE)
time.sleep(ADSL_CYCLE)
self.remove_proxy()
else:
print('Invalid Proxy')
time.sleep(5)
else:
print('Get IP Failed, Re Dialing')
time.sleep(ADSL_ERROR_CYCLE)
else:
print('ADSL Failed, Please Check')
time.sleep(ADSL_ERROR_CYCLE)