def run(self): try: log.setLevel(self.config['verbosity']) writer = self.config.subOptions.getTouRSStWriter() deferreds = [] for name, url in xbel.getRSSFeedsFromXBEL( filename=self.config['bookmark-file'], folderName=self.config['bookmark-folder'], ): log.verbose('Checking feed', name) feed=fetch.TouRSStFeed(name, url, mirrorDirectory=self.config['mirror']) feedWriter = writer.startWriting(feed) if feed.need_refresh() or self.config['force-refresh']: d = feed.refresh([feedWriter]) else: d = defer.succeed(None) d.addCallback(lambda _, feedWriter: feedWriter.stopWriting(), feedWriter) deferreds.append((feed, d)) dl = defer.DeferredList([d for feed, d in deferreds]) for feed, d in deferreds: d.addErrback(self._reportError, feed) dl.addBoth(lambda _, writer: writer.stopWriting(), writer) dl.addBoth(lambda _: reactor.callWhenRunning(reactor.stop)) reactor.run() except KeyboardInterrupt: print >>sys.stderr, '%s: interrupted.' % sys.argv[0] sys.exit(1)
def refresh(self, writerlist): log.verbose('Refreshing...') starttime=time.time() if self.mirrorDirectory is not None: def _slurp(url): import urllib host, port, path = client._parse(url) assert path.startswith('/'), 'URL path should start with a slash' path = path[1:] path = urllib.quote_plus(path, '?=/') if port != 80: host = '%s:%d' % (host, port) f = file(os.path.expanduser(os.path.join(self.mirrorDirectory, host, path))) data = f.read() return data d = defer.succeed(self.feed) d.addCallback(_slurp) # fall back to network use if local file is broken or missing d.addErrback(lambda _, url: client.getPage(url), self.feed) else: d = client.getPage(self.feed) d.addCallback(microdom.parseString, beExtremelyLenient=1) d.addCallback(RSSParser, starttime) d.addCallback(self._cbRefresh, writerlist, starttime) """ try: dom = makeBlocking(d) except (error.ConnectError, error.ConnectionLost), e: log.verbose('Connection broke: %s' % e) return except ValueError, (httpErrorCode, errorString): log.verbose('HTTP error %s: %s' % (httpErrorCode, errorString)) return rss = RSSParser(dom, starttime) """ return d