예제 #1
0
class ProxyCheck:
    def __init__(self):
        self._initial_proxies = None
        self._test_url = TEST_URL

    def get_initial_proxies(self, proxies):
        self._initial_proxies = proxies
        self._conn = RedisClient()

    async def test_single_proxy(self, proxy):
        try:
            async with aiohttp.ClientSession() as session:
                try:
                    if isinstance(proxy, bytes):
                        proxy = proxy.decode('utf-8')
                    current_proxy = "http://" + proxy
                    proxy_list = self._conn.get(self._conn.count())
                    async with session.get(self._test_url, proxy=current_proxy, timeout=10) as response:
                        if response.status == 200 and proxy not in proxy_list:
                            self._conn.put(proxy)
                            print("Valid proxy:", proxy)
                except (ProxyConnectionError, TimeoutError, ValueError):
                    print('Invalid proxy', proxy)
        except (ServerDisconnectedError, ClientResponseError, ClientConnectorError) as s:
            print(s)
            pass

    def test_all_proxies(self):
        loop = asyncio.get_event_loop()
        tasks = [self.test_single_proxy(proxy) for proxy in self._initial_proxies]
        result = loop.run_until_complete(asyncio.wait(tasks))
        print(result)
예제 #2
0
 def check_proxy(cycle=WAIT_TIME):
     conn = RedisClient()
     check = ProxyCheck()
     while True:
         count = int(0.5 * conn.count())
         if count == 0:
             print('there is no data in pool, please wait...')
             time.sleep(cycle)
         proxies = conn.get(count)
         check.get_initial_proxies(proxies)
         check.test_all_proxies()
         time.sleep(cycle)