def main():
"""主函数"""
p = ThreadPool(settings.THREAD_NUM)
p.run(verify, (
settings.BASE_URL,
p,
), callback=callback(p, ))
def startBatch(self, batch, **kwargs):
"""Process a batch"""
with self._mutex:
if self._running:
log.debug("can't start another batch - already running")
else:
self._running = True
self._batch = batch
self._pool = ThreadPool(name=self.name,
maxThreads=self._maxThreads())
# start the loading thread
t = threads.ModRanaThread(name=self.name+"Loader", target=self._loadItems)
self._loaderName = threads.threadMgr.add(t)
def download(bound, shared):
begin, end = bound
pool = ThreadPool(shared["num_thread"], fetch, shared["urls"][begin: end], shared)
pool.join()
