class Getter(object): def __init__(self): self.db = RedisClient() self.crawl = Crawler() def is_over_threshold(self): if self.db.count() >= POOL_UPPER_THRESHOLD: return True else: return False def run(self): if not self.is_over_threshold(): func_names = self.crawl.__FUNC__ for func_name in func_names: proxies = self.crawl.get_proxies(func_name) for proxy in proxies: self.db.add(proxy)
class Getter(object): def __init__(self): self.redis = RedisClient() self.crawler = Crawler() def is_over_threshold(self): if self.redis.count() >= POOL_UPPER_THRESHOLD: return True else: return False def run(self): print('获取器开始执行...') if not self.is_over_threshold(): for callback_label in range(self.crawler.__CrawlFuncCount__): callback = self.crawler.__CrawlFunc__[callback_label] proxies = self.crawler.get_proxies(callback) for proxy in proxies: self.redis.add(proxy) # r = Getter() # r.run()