Exemplo n.º 1
0
	def sendEnclosure(self, record):
		# A partial file may not exist if we encountered an error while trying to
		# fetch it.
		if os.path.exists(record.partialPath):
			partialFile = file(record.partialPath, 'rb')
			self.wfile.write(partialFile.read())
			skip = [partialFile.tell()]
			partialFile.close()
		else:
			skip = [0]
		url = record.url
		# TODO: forward any Range requests in headers
		RawHTTPClient.get(url, self.server.config.get('http_proxy'), self.server.rawServer, self.callback, record, skip)
Exemplo n.º 2
0
	def start(self, callback, *args, **kwargs):
		# Don't prefetch the enclosure if we already have it.
		if self.record.sent or self.record.numFailures > MAX_FAILURE or \
				(self.record.partialPath and os.path.exists(self.record.partialPath) and os.path.getsize(self.record.partialPath) >= 2048):
			callback(*args, **kwargs)
			return
		self.display.info('Prefetching first 2KB of enclosure: ' + self.url)
		self.client = RawHTTPClient.get(self.url, self.config.get('http_proxy'), self.rawServer, self.callback, callback, args, kwargs)
Exemplo n.º 3
0
	def fetchMore(self, callback, args, kwargs):
		if not self.enclosures:
			if not self.active:
				callback(*args, **kwargs)
			return
		url, enclosure = self.enclosures.pop(0)
		self.display.info('Fetching .torrent file: ' + url)
		dataBuffer = StringIO()
		client = RawHTTPClient.get(url, self.config.get('http_proxy'), self.rawServer, self.callback, url, dataBuffer, callback, args, kwargs)
		self.clients[url] = client
		self.active[url] = enclosure
Exemplo n.º 4
0
	def start(self, callback, *args, **kwargs):
		while self.enclosures and len(self.clients) < NUM_PREFETCHES:
			url, enclosure = self.enclosures.pop(0)
			# Don't refetch the .torrent unless we encountered problems fetching or if the
			# the published date has changed.
			record = self.feedStore.get(url)
			if record and record.torrentFile and os.path.exists(record.torrentFile) and record.numFailures == 0 and record.published == getPubDate(enclosure):
				f = file(record.torrentFile, 'rb')
				self.transformEnclosure(url, enclosure, f.read())
				f.close()
				continue
			# Don't bother to do anything with the enclosure if it has failed too often.
			if record and record.numFailures > MAX_FAILURE:
				# TODO: should we completely remove the <item> if we can't get the .torrent file?
				continue
			self.display.info('Fetching .torrent file: ' + url)
			dataBuffer = StringIO()
			client = RawHTTPClient.get(url, self.config.get('http_proxy'), self.rawServer, self.callback, url, dataBuffer, callback, args, kwargs)
			self.clients[url] = client
			self.active[url] = enclosure
		if not self.active:
			callback(*args, **kwargs)
			return
Exemplo n.º 5
0
	def fetch(self, rssUrl, writer, host):
		self.keepAlives[rssUrl] = FeedKeepAlive(writer, self.rawServer)
		self.keepAlives[rssUrl].start()
		dataBuffer = StringIO()
		client = RawHTTPClient.get(rssUrl, self.config.get('http_proxy'), self.rawServer, self.callback, dataBuffer, rssUrl, writer, host)
		self.clients[rssUrl] = client