class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
"""
测试单个代理
:param proxy: 单个代理
:return: None
"""
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print(' 正在测试 ', proxy)
async with session.get(TEST_URL,
proxy=real_proxy,
headers=HEADERS,
timeout=15) as response:
code = response.json().get('code')
print(response.json())
if response.status in VALID_STATUS_CODES and code in VALID_STATUS_CODES:
self.redis.max(proxy)
print(' 代理可用 ', proxy)
else:
self.redis.decrease(proxy)
print(' 请求响应码不合法, 不可用 ', proxy)
except (ClientError, ClientConnectorError, TimeoutError,
AttributeError):
self.redis.decrease(proxy)
print(' 代理请求失败 ', proxy)
def run(self):
"""
测试主函数
:return: None
"""
print(' 测试器开始运行 ')
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
# 批量测试
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print(' 测试器发生错误 ', e.args)
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
"""
测试单个代理
:param proxy:代理
:return: None
"""
conn = aiohttp.TCPConnector(ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('正在测试', proxy)
async with session.get(TEST_URL, proxy=real_proxy,
timeout=15) as resonse:
if resonse.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print('代理可用', proxy)
else:
self.redis.decrease(proxy)
print('请求响应码不合法', proxy)
except (ClientError, aiohttp.ClientProxyConnectionError,
asyncio.TimeoutError, AttributeError):
self.redis.decrease(proxy)
print('代理请求失败', proxy)
def run(self):
"""
测试主函数
:return: None
"""
print('检测器开始运行')
try:
proxies = self.redis.all()
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
#批量测试
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
asyncio.ensure_future(self.test_single_proxy(proxy))
for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print("检测器发生错误", e)
class Tester():
def __init__(self, test_url=url):
self.test_url = test_url
self.db = RedisClient()
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/84.0.4147.135 Safari/537.36 Edg/84.0.522.63',
'Origin':
'https://www.bilibili.com',
'accept-encoding':
'gzip, deflate, br',
'accept-language':
'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6'
}
def test_single_proxy(self, proxy):
proxies = {'http': 'http://' + proxy, 'https': 'https://' + proxy}
try:
response = requests.get(self.test_url,
headers=self.headers,
proxies=proxies,
timeout=5)
json = response.json()
code = json.get('data')
if code != None:
self.db.max(proxy)
print(' 代理可用 ', proxy)
else:
print(' 请求响应码不合法, 不可用 ', proxy)
self.db.decrease(proxy)
except:
print(' 代理请求失败 ', proxy)
self.db.decrease(proxy)
def run(self):
print(' 测试器开始运行 ')
proxies = self.db.all()
for proxy in proxies:
print(' 正在测试 ', proxy)
self.test_single_proxy(proxy)