class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
"""
测试单个代理
:param proxy: 单个代理
:return: None
"""
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('正在测试', proxy)
async with session.get(TEST_URL, proxy=real_proxy,
timeout=15) as response:
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print('代理可用', proxy)
else:
self.redis.decrease(proxy)
print('请求响应码不合法', proxy)
except:
self.redis.decrease(proxy)
print('代理请求失败', proxy)
def run(self):
"""
测试主函数
:return: None
"""
print('测试器开始运行')
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
# 批量测试
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print('测试器发生错误', e.args)
class ValidTester(object):
def __init__(self, website='default'):
self.website = website
self.cookies_db = RedisClient('cookies', self.website)
self.accounts_db = RedisClient('accounts', self.website)
self.header = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
}
# 测试cookies，这块用子类重写
def test(self, username, cookies):
raise NotImplementedError
# 获取所有得cookies进行测试
def run(self):
cookies_groups = self.cookies_db.all()
for username, cookies in cookies_groups.items():
self.test(username, cookies)
class Tester():
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode("utf-8")
http_proxy = "http://" + proxy
print("测试代理{}中".format(proxy))
async with session.get(TEXT_URL, proxy=http_proxy,
timeout=15) as response:
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print("代理可用", proxy)
else:
self.redis.decrease(proxy)
print("代理返回的响应码不合法", proxy)
except (ClientConnectionError, ClientError, AttributeError,
asyncio.TimeoutError):
self.redis.decrease(proxy)
def run(self):
print("测试开始运行")
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print("测试发生错误", e.args)
class TestCookies():
def __init__(self):
self.account_db = RedisClient('jdaccount')
self.cookies_db = RedisClient('jdcookies')
self.headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
}
self.testUrl = 'https://www.jd.com/'
def test(self, username, cookies):
print('开始测试账号为%s的cookies' % username)
try:
# 测试格式是否为json格式
json.dumps(cookies)
except TypeError:
print('cookies不合法', username)
self.cookies_db.delete(username)
print('删除cookies', username)
return
try:
# 加入cookies测试有效性
response = requests.get(self.testUrl,
headers=self.headers,
cookies=cookies,
timeout=5,
allow_redirects=False)
if response.status_code == 200:
print('账号%s的cookies有效' % username)
else:
print(response.status_code, response.headers)
print('账号%s的cookies已失效' % username)
self.cookies_db.delete(username)
print('删除cookies', username)
except ConnectionError as e:
print('发生异常', e.args)
def process_cookies(self, cookies):
dict = {}
# 提取cookies中的name和value组成新字典，可供requests调用的cookies
for cookie in cookies:
dict[cookie['name']] = cookie['value']
return dict
def run(self):
# 获取cookies表中的所有账号cookies
cookies_groups = self.cookies_db.all()
for username, cookies in cookies_groups.items():
# 将JSON格式转换为字典
a = json.loads(cookies)
# 将selenium生成的cookies转换为requests需要的格式进行测试
b = self.process_cookies(a)
self.test(username, b)
# a = CookiesGenerator()
# a = TestCookies()
# a.run()