def get(self):
task = self.request.get("task")
if task == "cleanup":
# Remove obsolete entries
Entry.cleanup()
elif task == "subscribe":
# Periodically make subscribe request
#
# Priorily subscribe to newly added channel (i.e. status == None);
# if there aren't any, then confirm the least checked subscription.
ch = Channel.all().filter("status =", None).get()
if not ch:
ch = Channel.all().filter("status =",
"subscribed").order("lastcheck").get()
ch.subscribe()
else: self.error(404)
def render_sec_featured(self):
"""Render featured section"""
query = Channel.all().filter("featured =", True)
featured_channels = query.fetch(6)
entries = [ch.latest_entry for ch in featured_channels
if ch.latest_entry]
path = templatepath("_featured.html")
return template.render(path, {"entries":entries})
def add_youtube_feeds(cls):
channels = Channel.all().fetch(100)
for channel in channels:
for keyword in channel.keywords:
yt_service = gdata.youtube.service.YouTubeService()
gdata.alt.appengine.run_on_appengine(yt_service)
uri = Programming.YOUTUBE_FEED % ('most_popular', keyword)
feed = yt_service.GetRecentlyFeaturedVideoFeed()
medias = Media.add_from_entry(feed.entry)
for media in medias:
Program.add_program(channel, media)
def clear_channels(cls):
channels = Channel.all().fetch(None)
memcache.delete('channels')
memcache.delete('programming')
for c in channels:
c.next_time = None
c.put()
chan_programs = c.programs.fetch(None)
for cp in chan_programs:
try:
cp.program.delete()
except:
pass
cp.delete()
def clear_model(cls, model):
channels = Channel.all().fetch(None)
memcache.delete('channels')
memcache.delete('programming')
for c in channels:
c.next_time = None
c.programming = []
c.put()
try:
while True:
q = db.GqlQuery("SELECT __key__ FROM " + model)
assert q.count()
db.delete(q.fetch(200))
time.sleep(0.5)
except Exception, e:
pass
def post(self):
"""Parsing queued feeds"""
doc = feedparser.parse(self.request.body)
# Bozo feed handling
# stealed from PubSubHubbub subscriber repo
if doc.bozo:
logging.error("Bozo feed data. %s: %r", doc.bozo_exception.__class__.__name__, doc.bozo_exception)
if hasattr(doc.bozo_exception, "getLineNumber") and hasattr(doc.bozo_exception, "getMessage"):
line = doc.bozo_exception.getLineNumber()
logging.error("Line %d: %s", line, doc.bozo_exception.getMessage())
segment = self.request.body.split("\n")[line - 1]
logging.info("Body segment with error: %r", segment.decode("utf-8"))
return # fail fast
# WORKER['parser'] + `key`
key = self.request.path[len(WORKER["parser"]) :]
# Try to get the channel by key;
# fallback to feed id, if not found;
# and at last we'll resort to entry source id,
# to find out the associated channel
channel = None
uid = doc.feed.id
try:
channel = Channel.get(key)
except:
channel = Channel.all().filter("uid =", uid).get()
else:
# First time get the notification,
# so update channel's properties
if channel and not channel.uid:
channel.title = doc.feed.title.split(" - ")[0]
channel.uid = uid
# Fallback to topic feed, if no link found
channel.link = doc.feed.get("link", channel.topic)
channel.put()
updates = []
for e in doc.entries:
author = e.author if e.get("author") else None
content = e.content[0].value if e.get("content") else e.summary
# Fallback to published if no updated field.
t = e.updated_parsed if e.get("updated_parsed") else e.published_parsed
updated = datetime(t[0], t[1], t[2], t[3], t[4], t[5])
# If we have this entry already in datastore, then the entry
# should be updated instead of inserted.
ent = Entry.all().filter("uid =", e.id).get()
if not ent:
if not channel:
uid = e.source.id
channel = Channel.all().filter("uid =", uid).get()
ent = Entry(
title=e.title,
link=e.link,
content=content,
author=author,
updated=updated,
uid=e.id,
channel=channel,
)
logging.info("Get new entry: %s" % e.id)
else:
ent.title = e.title
ent.link = e.link
ent.content = content
ent.author = author
ent.updated = updated
logging.info("Get updated entry: %s" % e.id)
updates.append(ent)
db.put(updates)
def get(self):
context = {}
context['channels'] = Channel.all().order("title").fetch(1000)
path = templatepath("following.html")
self.response.out.write(template.render(path, context))
