class ProxyCheck: def __init__(self): self._initial_proxies = None self._test_url = TEST_URL def get_initial_proxies(self, proxies): self._initial_proxies = proxies self._conn = RedisClient() async def test_single_proxy(self, proxy): try: async with aiohttp.ClientSession() as session: try: if isinstance(proxy, bytes): proxy = proxy.decode('utf-8') current_proxy = "http://" + proxy proxy_list = self._conn.get(self._conn.count()) async with session.get(self._test_url, proxy=current_proxy, timeout=10) as response: if response.status == 200 and proxy not in proxy_list: self._conn.put(proxy) print("Valid proxy:", proxy) except (ProxyConnectionError, TimeoutError, ValueError): print('Invalid proxy', proxy) except (ServerDisconnectedError, ClientResponseError, ClientConnectorError) as s: print(s) pass def test_all_proxies(self): loop = asyncio.get_event_loop() tasks = [self.test_single_proxy(proxy) for proxy in self._initial_proxies] result = loop.run_until_complete(asyncio.wait(tasks)) print(result)
def check_proxy(cycle=WAIT_TIME): conn = RedisClient() check = ProxyCheck() while True: count = int(0.5 * conn.count()) if count == 0: print('there is no data in pool, please wait...') time.sleep(cycle) proxies = conn.get(count) check.get_initial_proxies(proxies) check.test_all_proxies() time.sleep(cycle)