class ValidityTester(object):
test_api = TEST_API
def __init__(self):
self._raw_proxies = None
self._usable_proxies = []
def set_raw_proxies(self, proxies):
self._raw_proxies = proxies
self._conn = RedisClient()
def set_timing_params(self):
self._conn = RedisClient()
self._all_ips_item = self._conn.getAll() #把现在所有的ip列表都拿出来做检查
self._post_url = ALIE_API
async def test_single_proxy(self, proxy):
"""
text one proxy, if valid, put them to usable_proxies.
"""
try:
async with aiohttp.ClientSession() as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('Testing', proxy)
async with session.get(
self.test_api,
proxy=real_proxy,
timeout=get_proxy_timeout) as response:
if response.status == 200:
self._conn.put(proxy)
print('Valid proxy', proxy)
except (ProxyConnectionError, TimeoutError, ValueError):
print('Invalid proxy', proxy)
except (ServerDisconnectedError, ClientResponseError,
ClientConnectorError) as s:
print(s)
pass
def test(self):
"""
aio test all proxies.
"""
print('ValidityTester is working')
try:
loop = asyncio.get_event_loop()
tasks = [
self.test_single_proxy(proxy) for proxy in self._raw_proxies
] #test_single_proxy 检验ip是否有效
loop.run_until_complete(asyncio.wait(tasks))
#loop.run_until_complete(asyncio.gather(self.test_single_proxy(proxy) for proxy in self._raw_proxies))
except ValueError:
print('Async Error')
async def TimingCheckFunction(self, proxy):
try:
async with aiohttp.ClientSession() as session:
try:
if isinstance(proxy, bytes): #bytes=str
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
headers = {'User-Agent': choice(AGENTS)}
print('Timing Check Async Ip:' + str(proxy))
async with session.get(self._post_url,
proxy=real_proxy,
timeout=get_proxy_timeout,
headers=headers) as response:
if (response.status != 200):
self._conn.delete(proxy)
print('Delete Old Invalid Proxy', proxy)
else:
print('Keep Save IP', proxy)
except (ProxyConnectionError, TimeoutError, ValueError):
print('Foreach Delete Invalid Proxy Error', proxy)
self._conn.delete(proxy)
except (ServerDisconnectedError, ClientResponseError,
ClientConnectorError) as s:
print('-------')
print(s)
#self._conn.delete(proxy)
pass
def TimingCheck(self):
try:
loop = asyncio.get_event_loop()
tasks = [
self.TimingCheckFunction(proxy) for proxy in self._all_ips_item
] #test_single_proxy 检验ip是否有效
loop.run_until_complete(asyncio.wait(tasks))
except ValueError:
print('Timing Check Error')
