def __init__(self, ex_ev): if USE_PROXY: self.proxy_gen = ProxyManager( log=None , ok_timeout=30, ban_timeout=1000) self.workers = [] self.exit_event = ex_ev
def __init__(self, ex_ev): self.log = init_logger(self.__class__.__name__) if USE_PROXY: self.proxy_gen = ProxyManager( self.log, ok_timeout=30, ban_timeout=1000) self.workers = [] self.exit_event = ex_ev
def __init__(self, exit_event): self.exit_event = exit_event self.loop = asyncio.get_event_loop() self.queue_from_web = RabbitManager() self.inner_queue = queue.Queue() self.proxy_manager = ProxyManager() print(current_process().name) self.log = logging.getLogger(current_process().name) self.url = URL self.headers = HEADERS self.connect = None self.workers_count = 1 for w_num in range(self.workers_count): self.loop.create_task(self.fetch(w_num)) self.loop.create_task(self.listener()) self.loop.create_task(self.init_db()) self.log.info('crawler inited')