def main():
"""主函数"""
p = ThreadPool(settings.THREAD_NUM)
p.run(verify, (
settings.BASE_URL,
p,
), callback=callback(p, ))
def startBatch(self, batch, **kwargs):
"""Process a batch"""
with self._mutex:
if self._running:
log.debug("can't start another batch - already running")
else:
self._running = True
self._batch = batch
self._pool = ThreadPool(name=self.name,
maxThreads=self._maxThreads())
# start the loading thread
t = threads.ModRanaThread(name=self.name+"Loader", target=self._loadItems)
self._loaderName = threads.threadMgr.add(t)
class BatchPool(object):
def __init__(self, name=None):
if name is None:
self._name = _getBatchPoolName()
else:
self._name = name
self._batch = set()
self._doneCount = 0
self._running = False
self._shutdown = False
self._mutex = threading.RLock()
self._loaderName = None
self._pool = None
self.batchDone = Signal()
def startBatch(self, batch, **kwargs):
"""Process a batch"""
with self._mutex:
if self._running:
log.debug("can't start another batch - already running")
else:
self._running = True
self._batch = batch
self._pool = ThreadPool(name=self.name,
maxThreads=self._maxThreads())
# start the loading thread
t = threads.ModRanaThread(name=self.name+"Loader", target=self._loadItems)
self._loaderName = threads.threadMgr.add(t)
@property
def name(self):
return self._name
@property
def batchSize(self):
return len(self._batch)
@property
def done(self):
with self._mutex:
return self._doneCount
@property
def running(self):
with self._mutex:
return self._running
def _loadItems(self):
self._processBatch()
# in this moment we either run out of work, in which case
# we tell the pool to shutdown once all work is processed
# (now==False) or the batch pool has been explicitly shutdown
# (self._shutdown=True) in which case we tell the pool to shutdown
# as quickly as possible (now==True)
self._pool.shutdown(now=self._shutdown, join=True, async=False,
callback=self._stoppedCallback)
log.info("%s loader done", self.name)
def _handleItemWrapper(self, item):
self._handleItem(item)
# one item processed
with self._mutex:
self._doneCount+=1
def stop(self):
"""Stop processing as soon as possible"""
# tell the loading thread to shutdown
with self._mutex:
if self._running:
self._shutdown = True
def _stoppedCallback(self):
"""Called from a thread once the thread pool is fully stopped"""
# lets miss-use the callback thread a bit and make it wait for the
# loader to finish
self._shutdown = True
# now we can report we are no longer running & ready for new batch
with self._mutex:
# also cleanup so we are back to initial state
self._cleanup()
self._running = False
self._shutdown = False
# trigger the batch done signal
self.batchDone()
def _cleanup(self):
self._batch = []
self._doneCount = 0
self._loaderName = None
self._pool = None
# subclassing interface
def _processBatch(self):
pass
def _handleItem(self, item):
pass
def _maxThreads(self):
return 5
def download(bound, shared):
begin, end = bound
pool = ThreadPool(shared["num_thread"], fetch, shared["urls"][begin: end], shared)
pool.join()
