class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
"""
测试单个代理
:param proxy:
:return:
"""
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('正在验证 --> ', real_proxy)
async with session.get(TEST_URL, proxy=real_proxy,
timeout=15) as response:
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print('验证成功 -->', proxy)
else:
self.redis.decrease(proxy)
print('验证失败 Code 不合法-->', proxy)
except (ClientError,
aiohttp.client_exceptions.ClientConnectionError,
asyncio.TimeoutError, AttributeError):
self.redis.decrease(proxy)
print('验证失败 -->', proxy)
def run(self):
"""
测试主函数
:return:
"""
print("验证程序启动")
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy=proxy)
for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(3)
except Exception as e:
print('测试中发生了错误 --> ', e.args)
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
conn = aiohttp.TCPConnector(ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
async with session.get(TEST_URL,
proxy=real_proxy,
timeout=15,
allow_redirects=False) as response:
print('正在测试', proxy)
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print('代理可用', proxy)
else:
self.redis.decrease(proxy)
print('请求响应码不合法', proxy)
except (ClientError,
aiohttp.client_exceptions.ClientConnectorError,
TimeoutError, AttributeError):
self.redis.decrease(proxy)
print('代理请求失败', proxy)
def run(self):
proxy_sum = self.redis.count()
proxies = self.redis.all()
print('待检测代理个数', proxy_sum)
try:
for batch_limit in range(0, proxy_sum, BATCH_TEST_SIZE):
start = batch_limit
stop = min(batch_limit + BATCH_TEST_SIZE, proxy_sum)
print('当前正在检测代理第', start + 1, '至第', stop, '个')
proxy_batch = proxies[start:stop]
loop = asyncio.get_event_loop()
tasks = [
self.test_single_proxy(proxy) for proxy in proxy_batch
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print('测试器发生错误', e.args)
class Tester(object):
def __init__(self):
self.client = RedisClient()
async def test(self, proxy):
"""
测试单个proxy
:param proxy:
:return: None
"""
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
proxy_data = 'http://' + proxy
async with session.get(TEST_URL, proxy=proxy_data,
timeout=15) as response:
logger.info('测试：%s，结果：%s', proxy_data, response.status)
if response.status in VALID_STATUS:
logger.info('代理测试可用')
self.client.set_vaild(proxy)
else:
logger.info('代理测试不可用，分值减一')
self.client.decrease(proxy)
except Exception as e:
logger.info('%s 测试失败，分值减一', proxy)
self.client.decrease(proxy)
def run(self):
"""
批量测试代理
:return:
"""
logger.info('开始测试...')
try:
proxies = self.client.all()
loop = asyncio.get_event_loop()
for i in range(0, len(proxies), TEST_BATCH_SIZE):
test_proxy = proxies[i:i + TEST_BATCH_SIZE]
tester_list = [self.test(proxy) for proxy in test_proxy]
loop.run_until_complete(asyncio.wait(tester_list))
sleep(5)
except Exception as e:
logger.info('测试发生错误')
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
'''
测试单个代理k可用q情况，参数j就是被j检测的代理
:param proxy: 单个代理
:return: None
'''
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes)
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('正在测试 ', proxy)
async with session.get(TEST_URL, proxy=real_proxy, timeout=15) as response:
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print('代理可用 ', proxy)
else:
self.redis.decrease(proxy)
print('请求相应码不合法', proxy)
except (ServerDisconnectedError, ClientResponseError,ClientConnectorError):
self.redis.decrease(proxy)
print('代理请求失败', proxy)
def run(self):
'''
c测试z主函数
:return: None
'''
print('测试器开始运行')
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
#批量测试
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [self.test_single_proxy(proxy) for proxy in test_proxies]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print('c测试器f发生c错误', e.args)
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('Testing:', proxy)
async with session.get(TEST_URL, proxy=real_proxy,
timeout=15) as response:
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print('代理可用', proxy)
else:
self.redis.decrease(proxy)
print('请求响应码非法', proxy)
except (ClientError,
aiohttp.client_exceptions.ClientConnectorError,
asyncio.TimeoutError, AttributeError):
self.redis.decrease(proxy)
print('代理请求失败', proxy)
def run(self):
print('测试器开始运行')
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
#批量测试
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print('测试器发生错误', e.args)
class Tester():
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode("utf-8")
real_proxy = 'http://' + proxy
print("testing:", proxy)
async with session.get(TEST_URL, proxy=real_proxy,
timeout=15) as response:
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print("the proxy is avaliable:", proxy)
else:
self.redis.decrease(proxy)
print("the request code is not valid:", proxy)
except (ClientError,
aiohttp.client_exceptions.ClientConnectorError,
asyncio.TimeoutError, AttributeError):
self.redis.decrease(proxy)
print("failed to request the proxy:", proxy)
def run(self):
print("test is run")
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
for i in range(0, len(proxies), BACTH_TEST_SIZE):
test_proxies = proxies[i:i + BACTH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print("Tester error", e.args)