def fetcher():
while True:
print "Fetching"
try:
commands = []
while True:
cmd = fetcher_in.get(5)
if cmd not in commands:
commands.insert(0, cmd)
if fetcher_in.empty():
break
while len(commands):
cmd = commands.pop()
if cmd[0] == "update":
print "Updating:", cmd[1], "...",
f = feedparser.parse(cmd[1], etag=cmd[2], modified=cmd[3].timetuple())
if "bozo_exception" in f:
f["bozo_exception"] = None
fetcher_out.put(["updated", cmd[1], backend.sanitize(f)])
print "Done"
elif cmd[0] == "add":
print "Adding:", cmd[1], "..."
f = feedparser.parse(cmd[1])
if "bozo_exception" in f:
f["bozo_exception"] = None
fetcher_out.put(["added", cmd[1], backend.sanitize(f)])
print "Done"
except Exception as e:
print "exception in fetcher:", e
fetcher_out.put(["updated", cmd[1], {}])
def addPosts(self, feed=None):
'''Takes an optional already parsed feed'''
self.check_date = datetime.datetime.now()
saveData()
if feed == None:
feed=feedparser.parse(self.xmlurl,
etag = self.etag,
modified = self.check_date.timetuple())
elif feed == {}:
# This was probably a feedparser bug that made the
# fetcher crash, so don't try to do much, but
# mark as updated anyway
saveData()
return
# Fill in missing things
if not self.url:
if 'link' in feed['feed']:
self.url = feed['feed']['link']
elif 'links' in feed['feed'] and feed['feed']['links']:
self.url = feed['feed']['links'][0].href
# Keep data fresh
self.data = unicode(base64.b64encode(pickle.dumps(sanitize(feed['feed']))))
if 'status' in feed:
self.last_status = feed.status
if str(feed.status)[0]=='4':
# Error
self.bad_check_count+=1
else: #good check
self.last_good_check=datetime.datetime.now()
if feed.status == 304: # No change
print "Got 304 on feed update"
saveData()
return
elif feed.status == 301: # Permanent redirect
print "Got 301 on feed update => %s"%feed.href
self.xmlUrl=feed.href
elif feed.status == 410: # Feed deleted. FIXME: tell the user and stop trying!
print "Got 410 on feed update"
saveData()
return
elif feed.status == 404: # Feed gone. FIXME: tell the user and stop trying!
print "Got 404 on feed update"
saveData()
return
if 'etag' in feed:
self.etag = feed['etag']
self.oldest_fresh=datetime.datetime.now()
for post in feed['entries']:
p=Post.get_or_create(post)
self.posts.append(p)
if p.date < self.oldest_fresh:
self.oldest_fresh = p.date
saveData()
def on_actionNew_Feed_triggered(self, b=None):
"""Ask for site or feed URL and add it to backend"""
# FIXME: this is silly slow and blocking.
if b is not None:
return
url, r = QtGui.QInputDialog.getText(self, "Kakawana - New feed", "Enter the URL for the site")
if not r:
return
url = unicode(url)
feeds = []
feedurls = feedfinder.feeds(url)
if not feedurls:
# Didn't find any feeds
QtGui.QMessageBox.critical(
self, self.tr("Kakawana - Error"), self.tr("Couldn't find any feeds at that URL.")
)
return
for furl in feedurls:
f = feedparser.parse(furl)
feeds.append(f)
if len(feeds) > 1:
items = [u"%d - %s" % (i, feed["feed"]["title"]) for i, feed in enumerate(feeds)]
ll = QtCore.QStringList()
for i in items:
ll.append(QtCore.QString(i))
item, ok = QtGui.QInputDialog.getItem(
self, u"Kakawana - New feed", u"What feed do you prefer for this site?", ll, editable=False
)
if not ok:
return
# Finally, this is the feed URL
feed = feeds[items.index(unicode(item))]
furl = feedurls[items.index(unicode(item))]
else:
feed = feeds[0]
furl = feedurls[0]
f = backend.Feed.createFromFPData(furl, feed)
f.addPosts(feed=feed)
self.loadFeeds(f.xmlurl)
if self.keepGoogleSynced:
# Add this feed to google reader
reader = self.getGoogleReader2()
if reader:
print "Adding to google:", f.xmlurl, f.name
reader.subscribe_feed(f.xmlurl, f.name)
