def __init__(self): CrawlerBase.__init__(self) self.task_queue = queue.Queue() self.lock = threading.Lock()
def savePageContent(self, data): self.lock.acquire() c = CrawlerBase.savePageContent(self, data) self.total_new = self.total_new + c self.lock.release() return c
def __init__(self): CrawlerBase.__init__(self) # 使用长连接,所有请求都用一个 session self.session = aiohttp.ClientSession()
def __init__(self): CrawlerBase.__init__(self)
def singleRequest(self, i): return CrawlerBase.singleRequest(self, i)
def hasDuplicate(self, data): return CrawlerBase.hasDuplicate(self, data)
def addRecord(self, data): return CrawlerBase.addRecord(self, data)
def fetchPageContent(self, post={}): return CrawlerBase.fetchPageContent(self, post)