Beispiel #1
0
class Tester(object):
    """代理可用性测试器"""
    def __init__(self, proxy_key=setting.REDIS_KEY):
        self.redis = RedisClient(proxy_key=proxy_key)

    async def test_single_proxy(self, proxy):
        """
        测试单个代理
        :param proxy:
        :return:
        """
        conn = aiohttp.TCPConnector(verify_ssl=False)
        async with aiohttp.ClientSession(connector=conn) as session:
            try:
                if isinstance(proxy, bytes):
                    proxy = proxy.decode('utf-8')
                if not proxy.startswith('http://'):
                    proxy = 'http://' + proxy
                async with session.get(setting.TEST_URL,
                                       headers=setting.REQUEST_HEADERS,
                                       proxy=proxy,
                                       timeout=5,
                                       allow_redirects=False) as response:
                    if response.status in setting.VALID_STATUS_CODES:
                        self.redis.set_proxy_max(proxy)
                        logger.info('代理 {} 请求成功,响应码为 {}'.format(
                            proxy, response.status))
                    else:
                        self.redis.decrease_proxy(proxy)
                        logger.info('代理 {} 的响应码不合法,为 {}'.format(
                            proxy, response.status))
            except (aiohttp.ClientError,
                    aiohttp.client_exceptions.ClientConnectorError,
                    asyncio.TimeoutError, AttributeError) as e:
                self.redis.decrease_proxy(proxy)
                logger.warning('代理 {} 请求出错:{}'.format(proxy, repr(e)))

    def run(self):
        """
        测试主函数
        :return:
        """
        logger.info('测试器开始运行')
        try:
            count = self.redis.get_count()
            logger.info('当前有 {} 个代理'.format(count))
            for i in range(0, count, setting.BATCH_TEST_SIZE):
                start = i
                stop = min(i + setting.BATCH_TEST_SIZE, count)
                logger.info('正在测试第 {}-{} 个代理'.format(start + 1, stop))
                test_proxies = self.redis.get_batch(start, stop)
                loop = asyncio.get_event_loop()
                tasks = [
                    self.test_single_proxy(proxy) for proxy in test_proxies
                ]
                loop.run_until_complete(asyncio.wait(tasks))
                sys.stdout.flush()
                time.sleep(5)
        except Exception as e:
            logger.error('测试器发生错误')
Beispiel #2
0
class TestRedisClient(unittest.TestCase):
    def __init__(self, *args, **kwargs):
        super(TestRedisClient, self).__init__(*args, **kwargs)
        self.redis = RedisClient(proxy_key=TEST_REDIS_KEY)

    def setUp(self):
        self.redis.clear()
        print('')

    def tearDown(self):
        self.redis.clear()

    def test_add_proxy(self):
        self.redis.add_proxy('0.0.0.0:0', 10)
        self.redis.add_proxy('0.0.0.0', 10)
        self.assertTrue(self.redis.exist_proxy('0.0.0.0:0'))
        self.assertEqual(self.redis.get_count(), 1)

    def test_get_random(self):
        count = 10
        for i in range(count):
            self.redis.add_proxy('0.0.0.0:' + str(i), i)
        self.redis.set_proxy_max('0.0.0.0:0')

        # 获取到的是分数最大的代理
        self.assertEqual(self.redis.get_random(), '0.0.0.0:0')

        # 不存在分数最大的代理时,随机获取前100名的代理
        times = 5
        self.redis.decrease_proxy('0.0.0.0:0')
        print('不存在分数最大代理时,随机获取', times, '个代理:')
        for i in range(times):
            print(self.redis.get_random())