def get_conn():
"""获取 Redis 连接
:return: RedisOperator
"""
if not hasattr(g, 'redis_connect'):
g.redis_connect = RedisOperator()
return g.redis_connect
def run(self):
logging.config.dictConfig(self._logging_conf)
logger = logging.getLogger('root')
logger.info('进程1 - 代理数量监控启动，每%s秒检查一次' % self._cycle)
adder = PoolAdder()
pool = RedisOperator()
while True:
if pool.usable_size < self._lower_threshold:
adder.add_to_pool()
time.sleep(self._cycle)
def run(self):
logging.config.dictConfig(self._logging_conf)
logger = logging.getLogger('root')
logger.info('进程2 - 代理定时测试启动，每%s秒测试一次' % self._cycle)
tester = UsabilityTester()
pool = RedisOperator()
while True:
logger.debug('周期性测试开始，将对所有代理进行测试')
test_proxies = pool.get_all()
test_total = len(test_proxies)
if test_total < self._lower_threshold:
logger.debug('池中代理数量低于阈值，本次不进行测试')
time.sleep(self._cycle)
continue
tester.test(test_proxies)
after_test_total = pool.usable_size
logger.info('淘汰了 %s 个代理，现可用 %s 个代理' %
(test_total - after_test_total, after_test_total))
logger.info('本次测试结束，%s秒后再次测试' % self._cycle)
time.sleep(self._cycle)
def __init__(self):
self.proxies_arg = None
self._logger = logging.getLogger('root')
self._headers = HEADERS
self._pool = RedisOperator()
def __init__(self):
self._headers = HEADERS
self._logger = logging.getLogger('root')
self.test_api = TEST_API
self._pool = RedisOperator()
def __init__(self):
self._threshold = POOL_UPPER_THRESHOLD
self._pool = RedisOperator()
self._tester = UsabilityTester()
self._logger = logging.getLogger('root')