def get(self): # We need to clean up the url first and remove any fragment site_url = urlparse.urldefrag(self.request.get("url"))[0] force = (self.request.get("force").lower()) in ['true', '1'] extend = (self.request.get("extend").lower()) in ['true', '1'] feeds = [] # default value if site_url: feeds = memcache.get(site_url + "." + str(extend)) if feeds is not None and not force: # good logging.debug("Memcache hit.") self.render_json(feeds) else: logging.debug("Memcache miss.") try: result = urlfetch.fetch(url=site_url, deadline=10) parser = LinkExtractor() parser.set_base_url(site_url) parser.feed(result.content) if parser.links: feeds = parser.links else: feeds = [] if not feeds: # Let's check if by any chance this is actually not a feed? data = feedparser.parse(result.content) if data.bozo == 0: feed = {'title': data.feed.get('title', ''), 'rel': 'self', 'type': 'application/atom+xml', 'href': site_url} links = data.feed.get('links', []) if extend: self.extend_feed(feed, links) feeds = [feed] else: if extend: for f in feeds: data = feedparser.parse(f['href']) links = data.feed.get('links', []) self.extend_feed(f, links) except: feeds = [] if not memcache.set(site_url + "." + str(extend), feeds, 86400): logging.error("Memcache set failed.") else: logging.debug("Memcache set.") self.render_json(feeds) else: self.response.write(template.render(os.path.join(os.path.dirname(__file__), 'templates', "index.html"), {}))
def get(self): # We need to clean up the url first and remove any fragment site_url = urlparse.urldefrag(self.request.get("url"))[0] force = self.request.get("force") feeds = [] # default value if site_url: feeds = memcache.get(site_url) if feeds is not None and not force: # good logging.debug("Memcache hit.") self.render_json(feeds) else: logging.debug("Memcache miss.") try: result = urlfetch.fetch(url=site_url, deadline=10) parser = LinkExtractor() parser.set_base_url(site_url) parser.feed(result.content) if parser.links: feeds = parser.links else: feeds = [] if not feeds: # Let's check if by any chance this is actually not a feed? data = feedparser.parse(result.content) mimeType = "application/atom+xml" href = site_url if re.match("atom", data.version): mimeType = "application/atom+xml" feeds = [{ 'title': data.feed.title, 'rel': 'self', 'type': mimeType, 'href': href }] except: feeds = [] if not memcache.set(site_url, feeds, 86400): logging.error("Memcache set failed.") else: logging.debug("Memcache set.") self.render_json(feeds) else: self.response.out.write( template.render( os.path.join(os.path.dirname(__file__), 'templates', "index.html"), {}))
def get(self): # We need to clean up the url first and remove any fragment site_url = urlparse.urldefrag(self.request.get("url"))[0] force = self.request.get("force") feeds = [] # default value if site_url: feeds = memcache.get(site_url) if feeds is not None and not force: # good logging.debug("Memcache hit.") self.render_json(feeds) else: logging.debug("Memcache miss.") try: result = urlfetch.fetch(url=site_url, deadline=10) parser = LinkExtractor() parser.set_base_url(site_url) parser.feed(result.content) if parser.links: feeds = parser.links else: feeds = [] if not feeds: # Let's check if by any chance this is actually not a feed? data = feedparser.parse(result.content) mimeType = "application/atom+xml" href = site_url if re.match("atom", data.version): mimeType = "application/atom+xml" feeds = [{'title': data.feed.title, 'rel': 'self', 'type': mimeType, 'href': href}] except: feeds = [] if not memcache.set(site_url, feeds, 86400): logging.error("Memcache set failed.") else: logging.debug("Memcache set.") self.render_json(feeds) else: self.response.out.write(template.render(os.path.join(os.path.dirname(__file__), 'templates', "index.html"), {}))
def get(self): feeds = [] if self.request.get("resource"): feeds = memcache.get(self.request.get("resource")) if feeds is not None: # good logging.debug("Memcache hit.") else: logging.debug("Memcache miss.") try: result = urlfetch.fetch(url=self.request.get("resource"), deadline=10) parser = LinkExtractor() parser.set_base_url(self.request.get("resource")) parser.feed(result.content) if parser.links: feeds = parser.links else: feeds = [] if not feeds: # Let's check if by any chance this is actually not a feed? data = feedparser.parse(result.content) mimeType = "application/atom+xml" href = self.request.get("resource") if re.match("atom", data.version): mimeType = "application/atom+xml" feeds = [{'title': data.feed.title, 'rel': 'self', 'type': mimeType, 'href': href}] except: feeds = [] if not memcache.set(self.request.get("resource"), feeds, 86400): logging.error("Memcache set failed.") else: logging.debug("Memcache set.") self.response.out.write(template.render(os.path.join(os.path.dirname(__file__), 'templates', "subscribe.html"), {'appname': appname, 'feeds': feeds}))