def fetcher(): while True: print "Fetching" try: commands = [] while True: cmd = fetcher_in.get(5) if cmd not in commands: commands.insert(0, cmd) if fetcher_in.empty(): break while len(commands): cmd = commands.pop() if cmd[0] == "update": print "Updating:", cmd[1], "...", f = feedparser.parse(cmd[1], etag=cmd[2], modified=cmd[3].timetuple()) if "bozo_exception" in f: f["bozo_exception"] = None fetcher_out.put(["updated", cmd[1], backend.sanitize(f)]) print "Done" elif cmd[0] == "add": print "Adding:", cmd[1], "..." f = feedparser.parse(cmd[1]) if "bozo_exception" in f: f["bozo_exception"] = None fetcher_out.put(["added", cmd[1], backend.sanitize(f)]) print "Done" except Exception as e: print "exception in fetcher:", e fetcher_out.put(["updated", cmd[1], {}])
def addPosts(self, feed=None): '''Takes an optional already parsed feed''' self.check_date = datetime.datetime.now() saveData() if feed == None: feed=feedparser.parse(self.xmlurl, etag = self.etag, modified = self.check_date.timetuple()) elif feed == {}: # This was probably a feedparser bug that made the # fetcher crash, so don't try to do much, but # mark as updated anyway saveData() return # Fill in missing things if not self.url: if 'link' in feed['feed']: self.url = feed['feed']['link'] elif 'links' in feed['feed'] and feed['feed']['links']: self.url = feed['feed']['links'][0].href # Keep data fresh self.data = unicode(base64.b64encode(pickle.dumps(sanitize(feed['feed'])))) if 'status' in feed: self.last_status = feed.status if str(feed.status)[0]=='4': # Error self.bad_check_count+=1 else: #good check self.last_good_check=datetime.datetime.now() if feed.status == 304: # No change print "Got 304 on feed update" saveData() return elif feed.status == 301: # Permanent redirect print "Got 301 on feed update => %s"%feed.href self.xmlUrl=feed.href elif feed.status == 410: # Feed deleted. FIXME: tell the user and stop trying! print "Got 410 on feed update" saveData() return elif feed.status == 404: # Feed gone. FIXME: tell the user and stop trying! print "Got 404 on feed update" saveData() return if 'etag' in feed: self.etag = feed['etag'] self.oldest_fresh=datetime.datetime.now() for post in feed['entries']: p=Post.get_or_create(post) self.posts.append(p) if p.date < self.oldest_fresh: self.oldest_fresh = p.date saveData()
def on_actionNew_Feed_triggered(self, b=None): """Ask for site or feed URL and add it to backend""" # FIXME: this is silly slow and blocking. if b is not None: return url, r = QtGui.QInputDialog.getText(self, "Kakawana - New feed", "Enter the URL for the site") if not r: return url = unicode(url) feeds = [] feedurls = feedfinder.feeds(url) if not feedurls: # Didn't find any feeds QtGui.QMessageBox.critical( self, self.tr("Kakawana - Error"), self.tr("Couldn't find any feeds at that URL.") ) return for furl in feedurls: f = feedparser.parse(furl) feeds.append(f) if len(feeds) > 1: items = [u"%d - %s" % (i, feed["feed"]["title"]) for i, feed in enumerate(feeds)] ll = QtCore.QStringList() for i in items: ll.append(QtCore.QString(i)) item, ok = QtGui.QInputDialog.getItem( self, u"Kakawana - New feed", u"What feed do you prefer for this site?", ll, editable=False ) if not ok: return # Finally, this is the feed URL feed = feeds[items.index(unicode(item))] furl = feedurls[items.index(unicode(item))] else: feed = feeds[0] furl = feedurls[0] f = backend.Feed.createFromFPData(furl, feed) f.addPosts(feed=feed) self.loadFeeds(f.xmlurl) if self.keepGoogleSynced: # Add this feed to google reader reader = self.getGoogleReader2() if reader: print "Adding to google:", f.xmlurl, f.name reader.subscribe_feed(f.xmlurl, f.name)