def __init__(self):
CrawlerBase.__init__(self)
self.task_queue = queue.Queue()
self.lock = threading.Lock()
def savePageContent(self, data):
self.lock.acquire()
c = CrawlerBase.savePageContent(self, data)
self.total_new = self.total_new + c
self.lock.release()
return c
def __init__(self):
CrawlerBase.__init__(self)
# 使用长连接，所有请求都用一个 session
self.session = aiohttp.ClientSession()
def __init__(self):
CrawlerBase.__init__(self)
def savePageContent(self, data):
self.lock.acquire()
c = CrawlerBase.savePageContent(self, data)
self.total_new = self.total_new + c
self.lock.release()
return c
def __init__(self):
CrawlerBase.__init__(self)
def singleRequest(self, i):
return CrawlerBase.singleRequest(self, i)
def __init__(self):
CrawlerBase.__init__(self)
self.task_queue = queue.Queue()
self.lock = threading.Lock()
def hasDuplicate(self, data):
return CrawlerBase.hasDuplicate(self, data)
def addRecord(self, data):
return CrawlerBase.addRecord(self, data)
def fetchPageContent(self, post={}):
return CrawlerBase.fetchPageContent(self, post)
def singleRequest(self, i):
return CrawlerBase.singleRequest(self, i)
def addRecord(self, data):
return CrawlerBase.addRecord(self, data)
def hasDuplicate(self, data):
return CrawlerBase.hasDuplicate(self, data)
def fetchPageContent(self, post={}):
return CrawlerBase.fetchPageContent(self, post)