class Getter: def __init__(self): self.redis = RedisClient() self.crawler = Crawler() def is_less_threshold(self): """ 判断是否达到了代理池限制 """ if self.redis.available_count( ) <= POOL_UPPER_AV_THRESHOLD or self.redis.count( ) <= POOL_UPPER_THRESHOLD: return True else: return False def run(self): print('获取器开始执行') if self.is_less_threshold(): for callback_label in range(self.crawler.__CrawlFuncCount__): callback = self.crawler.__CrawlFunc__[callback_label] # 获取代理 proxies = self.crawler.get_proxies(callback) sys.stdout.flush() for proxy in proxies: self.redis.add(proxy)