class ValidityTester(object): test_api = TEST_API def __init__(self): self._raw_proxies = None self._usable_proxies = [] def set_raw_proxies(self, proxies): self._raw_proxies = proxies self._conn = RedisClient() def set_timing_params(self): self._conn = RedisClient() self._all_ips_item = self._conn.getAll() #把现在所有的ip列表都拿出来做检查 self._post_url = ALIE_API async def test_single_proxy(self, proxy): """ text one proxy, if valid, put them to usable_proxies. """ try: async with aiohttp.ClientSession() as session: try: if isinstance(proxy, bytes): proxy = proxy.decode('utf-8') real_proxy = 'http://' + proxy print('Testing', proxy) async with session.get( self.test_api, proxy=real_proxy, timeout=get_proxy_timeout) as response: if response.status == 200: self._conn.put(proxy) print('Valid proxy', proxy) except (ProxyConnectionError, TimeoutError, ValueError): print('Invalid proxy', proxy) except (ServerDisconnectedError, ClientResponseError, ClientConnectorError) as s: print(s) pass def test(self): """ aio test all proxies. """ print('ValidityTester is working') try: loop = asyncio.get_event_loop() tasks = [ self.test_single_proxy(proxy) for proxy in self._raw_proxies ] #test_single_proxy 检验ip是否有效 loop.run_until_complete(asyncio.wait(tasks)) #loop.run_until_complete(asyncio.gather(self.test_single_proxy(proxy) for proxy in self._raw_proxies)) except ValueError: print('Async Error') async def TimingCheckFunction(self, proxy): try: async with aiohttp.ClientSession() as session: try: if isinstance(proxy, bytes): #bytes=str proxy = proxy.decode('utf-8') real_proxy = 'http://' + proxy headers = {'User-Agent': choice(AGENTS)} print('Timing Check Async Ip:' + str(proxy)) async with session.get(self._post_url, proxy=real_proxy, timeout=get_proxy_timeout, headers=headers) as response: if (response.status != 200): self._conn.delete(proxy) print('Delete Old Invalid Proxy', proxy) else: print('Keep Save IP', proxy) except (ProxyConnectionError, TimeoutError, ValueError): print('Foreach Delete Invalid Proxy Error', proxy) self._conn.delete(proxy) except (ServerDisconnectedError, ClientResponseError, ClientConnectorError) as s: print('-------') print(s) #self._conn.delete(proxy) pass def TimingCheck(self): try: loop = asyncio.get_event_loop() tasks = [ self.TimingCheckFunction(proxy) for proxy in self._all_ips_item ] #test_single_proxy 检验ip是否有效 loop.run_until_complete(asyncio.wait(tasks)) except ValueError: print('Timing Check Error')