def __init__(self, ex_ev):
if USE_PROXY:
self.proxy_gen = ProxyManager(
log=None , ok_timeout=30, ban_timeout=1000)
self.workers = []
self.exit_event = ex_ev
def __init__(self, ex_ev):
self.log = init_logger(self.__class__.__name__)
if USE_PROXY:
self.proxy_gen = ProxyManager(
self.log, ok_timeout=30, ban_timeout=1000)
self.workers = []
self.exit_event = ex_ev
def __init__(self, exit_event):
self.exit_event = exit_event
self.loop = asyncio.get_event_loop()
self.queue_from_web = RabbitManager()
self.inner_queue = queue.Queue()
self.proxy_manager = ProxyManager()
print(current_process().name)
self.log = logging.getLogger(current_process().name)
self.url = URL
self.headers = HEADERS
self.connect = None
self.workers_count = 1
for w_num in range(self.workers_count):
self.loop.create_task(self.fetch(w_num))
self.loop.create_task(self.listener())
self.loop.create_task(self.init_db())
self.log.info('crawler inited')