def __init__(self):
"""
初始化logger, redis_conn
"""
super(ProxyFactory, self).__init__()
sys.path.insert(0, self.current_dir)
self.proxies_check_in_set = ThreadSafeSet()
self.proxies_check_out_set = TreadSafeDict()
self.load_site(proxy_site_spider)
self.load_site(self.args.spider_module)
self.redis_conn = Redis(self.settings.get("REDIS_HOST"), self.settings.get_int("REDIS_PORT"))
if self.args.check_method:
self.check_method = partial(load_function(self.args.check_method), self)
def __init__(self, settings, fetch_modules=None, check_method=None):
"""
初始化logger, redis_conn
"""
super(ProxyFactory, self).__init__()
sys.path.insert(0, self.current_dir)
self.settings = self.setting_wrapper.load(settings, default_settings)
self.logger = Logger.init_logger(self.settings, name=self.name)
self.proxies_check_in_queue = Queue()
self.proxies_check_out_queue = Queue()
self.load_site(proxy_site_spider)
self.load_site(fetch_modules)
self.redis_conn = Redis(self.settings.get("REDIS_HOST"),
self.settings.get_int("REDIS_PORT"))
if check_method:
self.check_method = partial(load_function(check_method), self)