def sendEnclosure(self, record): # A partial file may not exist if we encountered an error while trying to # fetch it. if os.path.exists(record.partialPath): partialFile = file(record.partialPath, 'rb') self.wfile.write(partialFile.read()) skip = [partialFile.tell()] partialFile.close() else: skip = [0] url = record.url # TODO: forward any Range requests in headers RawHTTPClient.get(url, self.server.config.get('http_proxy'), self.server.rawServer, self.callback, record, skip)
def start(self, callback, *args, **kwargs): # Don't prefetch the enclosure if we already have it. if self.record.sent or self.record.numFailures > MAX_FAILURE or \ (self.record.partialPath and os.path.exists(self.record.partialPath) and os.path.getsize(self.record.partialPath) >= 2048): callback(*args, **kwargs) return self.display.info('Prefetching first 2KB of enclosure: ' + self.url) self.client = RawHTTPClient.get(self.url, self.config.get('http_proxy'), self.rawServer, self.callback, callback, args, kwargs)
def fetchMore(self, callback, args, kwargs): if not self.enclosures: if not self.active: callback(*args, **kwargs) return url, enclosure = self.enclosures.pop(0) self.display.info('Fetching .torrent file: ' + url) dataBuffer = StringIO() client = RawHTTPClient.get(url, self.config.get('http_proxy'), self.rawServer, self.callback, url, dataBuffer, callback, args, kwargs) self.clients[url] = client self.active[url] = enclosure
def start(self, callback, *args, **kwargs): while self.enclosures and len(self.clients) < NUM_PREFETCHES: url, enclosure = self.enclosures.pop(0) # Don't refetch the .torrent unless we encountered problems fetching or if the # the published date has changed. record = self.feedStore.get(url) if record and record.torrentFile and os.path.exists(record.torrentFile) and record.numFailures == 0 and record.published == getPubDate(enclosure): f = file(record.torrentFile, 'rb') self.transformEnclosure(url, enclosure, f.read()) f.close() continue # Don't bother to do anything with the enclosure if it has failed too often. if record and record.numFailures > MAX_FAILURE: # TODO: should we completely remove the <item> if we can't get the .torrent file? continue self.display.info('Fetching .torrent file: ' + url) dataBuffer = StringIO() client = RawHTTPClient.get(url, self.config.get('http_proxy'), self.rawServer, self.callback, url, dataBuffer, callback, args, kwargs) self.clients[url] = client self.active[url] = enclosure if not self.active: callback(*args, **kwargs) return
def fetch(self, rssUrl, writer, host): self.keepAlives[rssUrl] = FeedKeepAlive(writer, self.rawServer) self.keepAlives[rssUrl].start() dataBuffer = StringIO() client = RawHTTPClient.get(rssUrl, self.config.get('http_proxy'), self.rawServer, self.callback, dataBuffer, rssUrl, writer, host) self.clients[rssUrl] = client