def __init__(self, mqUrl, mqCls=None, dlCls=None, maxSize=5): self._maxSize = maxSize self._mqCls = mqCls and mqCls or RedisMQ self._dls = Group() self._mqInWorkers = Group() self._mqOutWorkers = Group() self._mqin = self._mqCls(mqUrl) self._mqout = self._mqCls(mqUrl + '&suffix=HTML') self._inq = Queue(15) self._outq = Queue(15) self._dlCls = dlCls and dlCls or DefaultDownloader
class DownloaderManager(PyThunderBase): """ """ def __init__(self, mqUrl, mqCls=None, dlCls=None, maxSize=5): self._maxSize = maxSize self._mqCls = mqCls and mqCls or RedisMQ self._dls = Group() self._mqInWorkers = Group() self._mqOutWorkers = Group() self._mqin = self._mqCls(mqUrl) self._mqout = self._mqCls(mqUrl + '&suffix=HTML') self._inq = Queue(15) self._outq = Queue(15) self._dlCls = dlCls and dlCls or DefaultDownloader def start(self): [self._dls.add(self._dlCls(self._inq, self._outq)) for _ in six.moves.range(self._maxSize)] [self._mqOutWorkers.add(MQLocalWorker(self._outq, self._mqout)) for _ in six.moves.range(self._maxSize - 3)] [self._mqInWorkers.add(MQLocalWorker(self._mqin, self._inq)) for _ in six.moves.range(self._maxSize - 3)] self.executeGroup('start') def stop(self): self.executeGroup('stop') def hold(self): self.executeGroup('hold') def resume(self): self.executeGroup('resume') def executeGroup(self, action): if action in ('resume', 'start'): getattr(self._dls, action)() getattr(self._mqOutWorkers, action)() getattr(self._mqInWorkers, action)() else: getattr(self._mqInWorkers, action)() getattr(self._dls, action)() getattr(self._mqOutWorkers, action)() def __del__(self): try: self.stop() except: pass @property def started(self): return self._mqInWorkers.started and self._mqOutWorkers.started and self._dls.started