def __init__(self, mqUrl, mqCls=None, dlCls=None, maxSize=5):
self._maxSize = maxSize
self._mqCls = mqCls and mqCls or RedisMQ
self._dls = Group()
self._mqInWorkers = Group()
self._mqOutWorkers = Group()
self._mqin = self._mqCls(mqUrl)
self._mqout = self._mqCls(mqUrl + '&suffix=HTML')
self._inq = Queue(15)
self._outq = Queue(15)
self._dlCls = dlCls and dlCls or DefaultDownloader
class DownloaderManager(PyThunderBase):
"""
"""
def __init__(self, mqUrl, mqCls=None, dlCls=None, maxSize=5):
self._maxSize = maxSize
self._mqCls = mqCls and mqCls or RedisMQ
self._dls = Group()
self._mqInWorkers = Group()
self._mqOutWorkers = Group()
self._mqin = self._mqCls(mqUrl)
self._mqout = self._mqCls(mqUrl + '&suffix=HTML')
self._inq = Queue(15)
self._outq = Queue(15)
self._dlCls = dlCls and dlCls or DefaultDownloader
def start(self):
[self._dls.add(self._dlCls(self._inq, self._outq)) for _ in six.moves.range(self._maxSize)]
[self._mqOutWorkers.add(MQLocalWorker(self._outq, self._mqout)) for _ in six.moves.range(self._maxSize - 3)]
[self._mqInWorkers.add(MQLocalWorker(self._mqin, self._inq)) for _ in six.moves.range(self._maxSize - 3)]
self.executeGroup('start')
def stop(self):
self.executeGroup('stop')
def hold(self):
self.executeGroup('hold')
def resume(self):
self.executeGroup('resume')
def executeGroup(self, action):
if action in ('resume', 'start'):
getattr(self._dls, action)()
getattr(self._mqOutWorkers, action)()
getattr(self._mqInWorkers, action)()
else:
getattr(self._mqInWorkers, action)()
getattr(self._dls, action)()
getattr(self._mqOutWorkers, action)()
def __del__(self):
try:
self.stop()
except:
pass
@property
def started(self):
return self._mqInWorkers.started and self._mqOutWorkers.started and self._dls.started