def addRequest(self, uri, file,exitfunc):
self.exitfunc=exitfunc
if len(self.deferreds) >= self.SIZE:
# wait for completion of all previous requests
DeferredList(self.deferreds
).addCallback(self._callback)
self.deferreds = []
# queue the request
deferred = Deferred()
self.requests.append((uri, file,deferred))
return deferred
else:
# execute the request now
#deferred = downloadPage(url, file)
host, port, url = MyUtils.parse(uri)
f = client.HTTPDownloader(uri, file)
f.deferred.addCallbacks(callback=self.exitfunc,
callbackArgs=(file,) )
self.deferreds.append(f.deferred)
reactor.connectTCP(host, port, f)
return f.deferred
def _callback(self):
if len(self.requests) >self.SIZE:
queue = self.requests[:self.SIZE]
self.requests = self.requests[self.SIZE:]
else:
queue = self.requests[:]
self.requests = []
# execute the requests
for (uri,file, deferredHelper) in queue:
host, port, url = MyUtils.parse(uri)
f = client.HTTPDownloader(uri, file)
f.deferred.addCallbacks(callback=self.exitfunc, callbackArgs=(file,))
self.deferreds.append(f.deferred)
reactor.connectTCP(host, port, f)
f.deferred.chainDeferred(deferredHelper)
