class ProxyCheck:
def __init__(self):
self._initial_proxies = None
self._test_url = TEST_URL
def get_initial_proxies(self, proxies):
self._initial_proxies = proxies
self._conn = RedisClient()
async def test_single_proxy(self, proxy):
try:
async with aiohttp.ClientSession() as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
current_proxy = "http://" + proxy
proxy_list = self._conn.get(self._conn.count())
async with session.get(self._test_url, proxy=current_proxy, timeout=10) as response:
if response.status == 200 and proxy not in proxy_list:
self._conn.put(proxy)
print("Valid proxy:", proxy)
except (ProxyConnectionError, TimeoutError, ValueError):
print('Invalid proxy', proxy)
except (ServerDisconnectedError, ClientResponseError, ClientConnectorError) as s:
print(s)
pass
def test_all_proxies(self):
loop = asyncio.get_event_loop()
tasks = [self.test_single_proxy(proxy) for proxy in self._initial_proxies]
result = loop.run_until_complete(asyncio.wait(tasks))
print(result)
def check_proxy(cycle=WAIT_TIME):
conn = RedisClient()
check = ProxyCheck()
while True:
count = int(0.5 * conn.count())
if count == 0:
print('there is no data in pool, please wait...')
time.sleep(cycle)
proxies = conn.get(count)
check.get_initial_proxies(proxies)
check.test_all_proxies()
time.sleep(cycle)