def get(self): task = self.request.get("task") if task == "cleanup": # Remove obsolete entries Entry.cleanup() elif task == "subscribe": # Periodically make subscribe request # # Priorily subscribe to newly added channel (i.e. status == None); # if there aren't any, then confirm the least checked subscription. ch = Channel.all().filter("status =", None).get() if not ch: ch = Channel.all().filter("status =", "subscribed").order("lastcheck").get() ch.subscribe() else: self.error(404)
def render_sec_featured(self): """Render featured section""" query = Channel.all().filter("featured =", True) featured_channels = query.fetch(6) entries = [ch.latest_entry for ch in featured_channels if ch.latest_entry] path = templatepath("_featured.html") return template.render(path, {"entries":entries})
def add_youtube_feeds(cls): channels = Channel.all().fetch(100) for channel in channels: for keyword in channel.keywords: yt_service = gdata.youtube.service.YouTubeService() gdata.alt.appengine.run_on_appengine(yt_service) uri = Programming.YOUTUBE_FEED % ('most_popular', keyword) feed = yt_service.GetRecentlyFeaturedVideoFeed() medias = Media.add_from_entry(feed.entry) for media in medias: Program.add_program(channel, media)
def clear_channels(cls): channels = Channel.all().fetch(None) memcache.delete('channels') memcache.delete('programming') for c in channels: c.next_time = None c.put() chan_programs = c.programs.fetch(None) for cp in chan_programs: try: cp.program.delete() except: pass cp.delete()
def clear_model(cls, model): channels = Channel.all().fetch(None) memcache.delete('channels') memcache.delete('programming') for c in channels: c.next_time = None c.programming = [] c.put() try: while True: q = db.GqlQuery("SELECT __key__ FROM " + model) assert q.count() db.delete(q.fetch(200)) time.sleep(0.5) except Exception, e: pass
def post(self): """Parsing queued feeds""" doc = feedparser.parse(self.request.body) # Bozo feed handling # stealed from PubSubHubbub subscriber repo if doc.bozo: logging.error("Bozo feed data. %s: %r", doc.bozo_exception.__class__.__name__, doc.bozo_exception) if hasattr(doc.bozo_exception, "getLineNumber") and hasattr(doc.bozo_exception, "getMessage"): line = doc.bozo_exception.getLineNumber() logging.error("Line %d: %s", line, doc.bozo_exception.getMessage()) segment = self.request.body.split("\n")[line - 1] logging.info("Body segment with error: %r", segment.decode("utf-8")) return # fail fast # WORKER['parser'] + `key` key = self.request.path[len(WORKER["parser"]) :] # Try to get the channel by key; # fallback to feed id, if not found; # and at last we'll resort to entry source id, # to find out the associated channel channel = None uid = doc.feed.id try: channel = Channel.get(key) except: channel = Channel.all().filter("uid =", uid).get() else: # First time get the notification, # so update channel's properties if channel and not channel.uid: channel.title = doc.feed.title.split(" - ")[0] channel.uid = uid # Fallback to topic feed, if no link found channel.link = doc.feed.get("link", channel.topic) channel.put() updates = [] for e in doc.entries: author = e.author if e.get("author") else None content = e.content[0].value if e.get("content") else e.summary # Fallback to published if no updated field. t = e.updated_parsed if e.get("updated_parsed") else e.published_parsed updated = datetime(t[0], t[1], t[2], t[3], t[4], t[5]) # If we have this entry already in datastore, then the entry # should be updated instead of inserted. ent = Entry.all().filter("uid =", e.id).get() if not ent: if not channel: uid = e.source.id channel = Channel.all().filter("uid =", uid).get() ent = Entry( title=e.title, link=e.link, content=content, author=author, updated=updated, uid=e.id, channel=channel, ) logging.info("Get new entry: %s" % e.id) else: ent.title = e.title ent.link = e.link ent.content = content ent.author = author ent.updated = updated logging.info("Get updated entry: %s" % e.id) updates.append(ent) db.put(updates)
def get(self): context = {} context['channels'] = Channel.all().order("title").fetch(1000) path = templatepath("following.html") self.response.out.write(template.render(path, context))