class WeiboValidTester(object):
def __init__(self, website='weibo'):
self.website = website
self.cookies_db = RedisClient('cookies', self.website)
self.uid_db = RedisClient('uid', self.website)
def test(self, username, cookies):
print('正在测试Cookies', '用户名', username)
try:
cookies = json.loads(cookies)
except TypeError:
print('Cookies不合法', username)
self.cookies_db.delete(username)
print('删除Cookies', username)
return
try:
test_url = TEST_URL_MAP[self.website]
response = requests.get(test_url % self.uid_db.get(username),
cookies=cookies,
timeout=5,
allow_redirects=False)
if response.status_code == 200:
print('Cookies有效', username)
else:
print(response.status_code, response.headers)
print('Cookies失效', username)
self.cookies_db.delete(username)
print('删除Cookies', username)
except ConnectionError as e:
print('发生异常', e.args)
def run(self):
cookies_groups = self.cookies_db.all()
for username, cookies in cookies_groups.items():
self.test(username, cookies)
def do_response():
redis = RedisClient(host=HOST, port=PORT, key=KEY)
RANDOM = str(redis.random())
RANDOM_MAX = str(redis.random_max())
ALL = redis.all()
all = ''
for ip in ALL:
all += str(ip) + "<br />"
ALL = all
ALL_MAX = redis.all_max()
all = ''
for ip in ALL_MAX:
all += str(ip) + "<br />"
ALL_MAX = all
COUNT = str(redis.count())
WRONG = 'Sorry,wrong name or wrong password,try again...'
if request.method == "GET":
name = request.args.get("name", "")
password = request.args.get("password", "")
method = request.args.get("method", "")
if not name:
return app.send_static_file("index.html")
if name == "yooongchun" and password == "121561":
if method == "random":
return RANDOM
elif method == "random_max":
return RANDOM_MAX
elif method == "all":
return ALL
elif method == "count":
return COUNT
elif method == "all_max":
return ALL_MAX
else:
return WRONG
else:
return WRONG
if request.method == "POST":
name = request.form['name']
password = request.form['password']
method = request.form['method']
if name == "yooongchun" and password == "121561":
if method == "random":
return RANDOM
elif method == "random_max":
return RANDOM_MAX
elif method == "all":
return ALL
elif method == "count":
return COUNT
elif method == "all_max":
return ALL_MAX
else:
return WRONG
else:
return WRONG
class Validation(object):
def __init__(self,
validation_addr="http://www.baidu.com",
host="www.baidu.com",
batch_size=100,
db_host='localhost',
port='6379',
key='Proxy'):
self.__redis = RedisClient(host=db_host,
port=port,
password=None,
key=key)
self.__host = host
self.__addr = validation_addr
self.__batch_size = batch_size
async def validation_single(self, proxy):
'''validate single proxy'''
# print("validation proxy: ", proxy)
headers = UserAgent(host=self.__host).headers()
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
async with session.get(self.__addr,
proxy=real_proxy,
headers=headers,
timeout=15) as response:
if response.status == 200:
self.__redis.max(proxy)
# print("proxy {} is good.".format(proxy))
else:
self.__redis.decrease(proxy)
# print("proxy {} is not good".format(proxy))
except Exception:
# print("proxy {} is not good".format(proxy))
self.__redis.decrease(proxy)
def run(self):
try:
proxies = self.__redis.all()
for i in range(0, len(proxies), self.__batch_size):
# print("processing i: {}\tprogress:{}/{} {:.2f}%".format(i + 1, i + 1, len(proxies),
# (i + 1) / len(proxies) * 100))
task_proxies = proxies[i:self.__batch_size + i]
loop = asyncio.get_event_loop()
tasks = [
self.validation_single(proxy) for proxy in task_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
# print("After validation,total {} in database".format(self.__redis.count()))
except Exception as e:
pass
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
"""
测试单个代理
:param proxy:
:return:
"""
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('正在测试：', proxy)
async with session.get(TEST_URL, proxy=real_proxy,
timeout=15) as response:
if response.status in VALID_STSTUS_CODES:
self.redis.max(proxy)
print('代理可用：', proxy)
else:
self.redis.decrease(proxy)
print('请求响应码不合法', proxy)
except (ClientError, ClientConnectorError, TimeoutError,
AttributeError):
self.redis.decrease(proxy)
print('代理请求失败', proxy)
def run(self):
"""
测试主函数
:return:
"""
print('测试器开始运行')
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
#批量测试
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print('测试器发生错误', e.args)
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
"""
测试单个代理
:param proxy: 单个代理
:return: NOne
"""
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = "http://" + proxy
print("正在测试", proxy)
async with session.get(TEST_URL, proxy=real_proxy,
timeout=15) as response:
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print("代理可用", proxy)
else:
self.redis.decrease(proxy)
print("请求接口相应不合法", proxy)
except:
pass
def run(self):
"""
测试主函数
:return: None
"""
print("测试器开始执行")
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
#批量测试
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print("测试器发生错误", e.args)
class API(object):
def __init__(self,
host="localhost",
port='6379',
key='Proxy',
moni_port=9999):
self.__redis = RedisClient(host=host,
port=port,
password=None,
key=key)
self.__port = moni_port
self.__host = host
self.__db_port = port
self.__key = key
def __message(self, method):
if method == "random":
return self.__redis.random()
elif method == "random_max":
return self.__redis.random_max()
elif method == "all":
return self.__redis.all()
elif method == "all_max":
return self.__redis.all_max()
elif method == "count":
return self.__redis.count()
else:
return "Sorry,wrong name or wrong password,try again..."
def run(self):
self.__set_api()
app.run(host="0.0.0.0", port=self.__port)
def __set_api(self):
global HOST
global PORT
global KEY
global MONI_PORT
HOST = self.__host
PORT = self.__db_port
KEY = self.__key
MONI_PORT = self.__port
class Tester(object):
def __init__(self):
self.redis = RedisClient()
# 异步的方法
async def test_single_proxy(self, proxy):
"""
方法用于检测一个代理是否合法
:param proxy: 需要检测的代理
:return:
"""
# 用来设置一次最大连接数量 参数用来防止ssl报错
conn = aiohttp.TCPConnector(verify_ssl=False)
# 用来创建一个Session连接
async with aiohttp.ClientSession(connector=conn) as session:
try:
# 检测proxy是否为bytes类型
if isinstance(proxy, bytes):
# 如果是的话 用utf-8进行proxy编码
proxy = proxy.decode('utf-8')
real_proxy = "http://" + proxy
print("testing...", proxy)
# 发起get请求
async with session.get(TEST_URL, proxy=real_proxy,
timeout=15) as response:
# 如果响应状态码是200
if response.status in VALID_STATUS_CODE:
# 将proxy的分数设置为 100
self.redis.max(proxy)
print("proxy ok", proxy)
else:
# 将代理分数减一
self.redis.decrease(proxy)
print("return code is illegal", proxy)
except (aiohttp.ClientError, aiohttp.ClientConnectorError,
TimeoutError, AttributeError):
self.redis.decrease(proxy)
print("proxy request fail", proxy)
def run(self):
"""
测试主函数
:return:
"""
print("测试器开始运行")
try:
proxies = self.redis.all()
# 创建消息循环队列
loop = asyncio.get_event_loop()
# 进行批量测试
for i in range(0, len(proxies), BATCH_TEST_SIZE):
# 一次测试 100 个代理
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print("error", e.args)
