コード例 #1
0
class Tester(object):
    def __init__(self):
        self.redis = RedisClient()

    async def test_single_proxy(self, proxy):
        """
        测试单个代理
        :param proxy:
        :return:
        """

        conn = aiohttp.TCPConnector(verify_ssl=False)
        async with aiohttp.ClientSession(connector=conn) as session:
            try:
                if isinstance(proxy, bytes):
                    proxy = proxy.decode('utf-8')

                real_proxy = 'http://' + proxy
                print('正在验证 --> ', real_proxy)
                async with session.get(TEST_URL, proxy=real_proxy,
                                       timeout=15) as response:
                    if response.status in VALID_STATUS_CODES:
                        self.redis.max(proxy)
                        print('验证成功 -->', proxy)
                    else:
                        self.redis.decrease(proxy)
                        print('验证失败 Code 不合法-->', proxy)

            except (ClientError,
                    aiohttp.client_exceptions.ClientConnectionError,
                    asyncio.TimeoutError, AttributeError):
                self.redis.decrease(proxy)
                print('验证失败 -->', proxy)

    def run(self):
        """
        测试主函数
        :return:
        """
        print("验证程序启动")
        try:
            proxies = self.redis.all()
            loop = asyncio.get_event_loop()
            for i in range(0, len(proxies), BATCH_TEST_SIZE):
                test_proxies = proxies[i:i + BATCH_TEST_SIZE]
                tasks = [
                    self.test_single_proxy(proxy=proxy)
                    for proxy in test_proxies
                ]
                loop.run_until_complete(asyncio.wait(tasks))
                time.sleep(3)
        except Exception as e:
            print('测试中发生了错误 --> ', e.args)
コード例 #2
0
class Tester(object):
    def __init__(self):
        self.redis = RedisClient()

    async def test_single_proxy(self, proxy):
        conn = aiohttp.TCPConnector(ssl=False)
        async with aiohttp.ClientSession(connector=conn) as session:
            try:
                if isinstance(proxy, bytes):
                    proxy = proxy.decode('utf-8')
                real_proxy = 'http://' + proxy
                async with session.get(TEST_URL,
                                       proxy=real_proxy,
                                       timeout=15,
                                       allow_redirects=False) as response:
                    print('正在测试', proxy)
                    if response.status in VALID_STATUS_CODES:
                        self.redis.max(proxy)
                        print('代理可用', proxy)
                    else:
                        self.redis.decrease(proxy)
                        print('请求响应码不合法', proxy)
            except (ClientError,
                    aiohttp.client_exceptions.ClientConnectorError,
                    TimeoutError, AttributeError):
                self.redis.decrease(proxy)
                print('代理请求失败', proxy)

    def run(self):
        proxy_sum = self.redis.count()
        proxies = self.redis.all()
        print('待检测代理个数', proxy_sum)
        try:
            for batch_limit in range(0, proxy_sum, BATCH_TEST_SIZE):
                start = batch_limit
                stop = min(batch_limit + BATCH_TEST_SIZE, proxy_sum)
                print('当前正在检测代理第', start + 1, '至第', stop, '个')
                proxy_batch = proxies[start:stop]
                loop = asyncio.get_event_loop()
                tasks = [
                    self.test_single_proxy(proxy) for proxy in proxy_batch
                ]
                loop.run_until_complete(asyncio.wait(tasks))
                time.sleep(5)
        except Exception as e:
            print('测试器发生错误', e.args)
コード例 #3
0
class Tester(object):
    def __init__(self):
        self.client = RedisClient()

    async def test(self, proxy):
        """
        测试单个proxy
        :param proxy:
        :return: None
        """
        conn = aiohttp.TCPConnector(verify_ssl=False)
        async with aiohttp.ClientSession(connector=conn) as session:
            try:
                if isinstance(proxy, bytes):
                    proxy = proxy.decode('utf-8')
                proxy_data = 'http://' + proxy
                async with session.get(TEST_URL, proxy=proxy_data,
                                       timeout=15) as response:
                    logger.info('测试:%s,结果:%s', proxy_data, response.status)
                    if response.status in VALID_STATUS:
                        logger.info('代理测试可用')
                        self.client.set_vaild(proxy)
                    else:
                        logger.info('代理测试不可用,分值减一')
                        self.client.decrease(proxy)
            except Exception as e:
                logger.info('%s 测试失败,分值减一', proxy)
                self.client.decrease(proxy)

    def run(self):
        """
        批量测试代理
        :return:
        """
        logger.info('开始测试...')
        try:
            proxies = self.client.all()
            loop = asyncio.get_event_loop()
            for i in range(0, len(proxies), TEST_BATCH_SIZE):
                test_proxy = proxies[i:i + TEST_BATCH_SIZE]
                tester_list = [self.test(proxy) for proxy in test_proxy]
                loop.run_until_complete(asyncio.wait(tester_list))
                sleep(5)
        except Exception as e:
            logger.info('测试发生错误')
コード例 #4
0
ファイル: tester.py プロジェクト: existenceE/into
class Tester(object):
    def __init__(self):
        self.redis = RedisClient()

    async def test_single_proxy(self, proxy):
        '''
        测试单个代理k可用q情况,参数j就是被j检测的代理
        :param proxy: 单个代理
        :return: None
        '''
        conn = aiohttp.TCPConnector(verify_ssl=False)
        async with aiohttp.ClientSession(connector=conn) as session:
            try:
                if isinstance(proxy, bytes)
                    proxy = proxy.decode('utf-8')
                real_proxy = 'http://' + proxy
                print('正在测试 ', proxy)
                async with session.get(TEST_URL, proxy=real_proxy, timeout=15) as response:
                    if response.status in VALID_STATUS_CODES:
                        self.redis.max(proxy)
                        print('代理可用 ', proxy)
                    else:
                        self.redis.decrease(proxy)
                        print('请求相应码不合法', proxy)
            except (ServerDisconnectedError, ClientResponseError,ClientConnectorError):
                self.redis.decrease(proxy)
                print('代理请求失败', proxy)

    def run(self):
        '''
        c测试z主函数
        :return: None
        '''
        print('测试器开始运行')
        try:
            proxies = self.redis.all()
            loop = asyncio.get_event_loop()
            #批量测试
            for i in range(0, len(proxies), BATCH_TEST_SIZE):
                test_proxies = proxies[i:i + BATCH_TEST_SIZE]
                tasks = [self.test_single_proxy(proxy) for proxy in test_proxies]
                loop.run_until_complete(asyncio.wait(tasks))
                time.sleep(5)
        except Exception as e:
            print('c测试器f发生c错误', e.args)
コード例 #5
0
class Tester(object):
    def __init__(self):
        self.redis = RedisClient()

    async def test_single_proxy(self, proxy):
        conn = aiohttp.TCPConnector(verify_ssl=False)
        async with aiohttp.ClientSession(connector=conn) as session:
            try:
                if isinstance(proxy, bytes):
                    proxy = proxy.decode('utf-8')
                real_proxy = 'http://' + proxy
                print('Testing:', proxy)
                async with session.get(TEST_URL, proxy=real_proxy,
                                       timeout=15) as response:
                    if response.status in VALID_STATUS_CODES:
                        self.redis.max(proxy)
                        print('代理可用', proxy)
                    else:
                        self.redis.decrease(proxy)
                        print('请求响应码非法', proxy)
            except (ClientError,
                    aiohttp.client_exceptions.ClientConnectorError,
                    asyncio.TimeoutError, AttributeError):
                self.redis.decrease(proxy)
                print('代理请求失败', proxy)

    def run(self):
        print('测试器开始运行')
        try:
            proxies = self.redis.all()
            loop = asyncio.get_event_loop()
            #批量测试
            for i in range(0, len(proxies), BATCH_TEST_SIZE):
                test_proxies = proxies[i:i + BATCH_TEST_SIZE]
                tasks = [
                    self.test_single_proxy(proxy) for proxy in test_proxies
                ]
                loop.run_until_complete(asyncio.wait(tasks))
                time.sleep(5)
        except Exception as e:
            print('测试器发生错误', e.args)
コード例 #6
0
class Tester():
    def __init__(self):
        self.redis = RedisClient()

    async def test_single_proxy(self, proxy):
        conn = aiohttp.TCPConnector(verify_ssl=False)
        async with aiohttp.ClientSession(connector=conn) as session:
            try:
                if isinstance(proxy, bytes):
                    proxy = proxy.decode("utf-8")
                real_proxy = 'http://' + proxy
                print("testing:", proxy)
                async with session.get(TEST_URL, proxy=real_proxy,
                                       timeout=15) as response:
                    if response.status in VALID_STATUS_CODES:
                        self.redis.max(proxy)
                        print("the proxy is avaliable:", proxy)
                    else:
                        self.redis.decrease(proxy)
                        print("the request code is not valid:", proxy)
            except (ClientError,
                    aiohttp.client_exceptions.ClientConnectorError,
                    asyncio.TimeoutError, AttributeError):
                self.redis.decrease(proxy)
                print("failed to request the proxy:", proxy)

    def run(self):
        print("test is run")
        try:
            proxies = self.redis.all()
            loop = asyncio.get_event_loop()
            for i in range(0, len(proxies), BACTH_TEST_SIZE):
                test_proxies = proxies[i:i + BACTH_TEST_SIZE]
                tasks = [
                    self.test_single_proxy(proxy) for proxy in test_proxies
                ]
                loop.run_until_complete(asyncio.wait(tasks))
                time.sleep(5)
        except Exception as e:
            print("Tester error", e.args)