Example #1
0
  def get(self):
    # We need to clean up the url first and remove any fragment
    site_url = urlparse.urldefrag(self.request.get("url"))[0]
    force = self.request.get("force")
    feeds = [] # default value
    if site_url:
      feeds = memcache.get(site_url)
      if feeds is not None and not force:
        # good
        logging.debug("Memcache hit.")
        self.render_json(feeds)
      else:
        logging.debug("Memcache miss.")
        try:
          result = urlfetch.fetch(url=site_url, deadline=10)
          parser = LinkExtractor()
          parser.set_base_url(site_url)
          parser.feed(result.content)
          if parser.links:
            feeds = parser.links
          else:
            feeds = []

          if not feeds:
              # Let's check if by any chance this is actually not a feed?
              data = feedparser.parse(result.content)
              mimeType = "application/atom+xml"
              href = site_url
              if re.match("atom", data.version):
                  mimeType = "application/atom+xml"
              feeds = [{'title': data.feed.title, 'rel': 'self', 'type': mimeType, 'href': href}]

        except:
          feeds = []

        if not memcache.set(site_url, feeds, 86400):
          logging.error("Memcache set failed.")
        else:
          logging.debug("Memcache set.")
        self.render_json(feeds)

    else:
      self.response.out.write(template.render(os.path.join(os.path.dirname(__file__), 'templates', "index.html"), {}))
Example #2
0
  def get(self):
    feeds = []
    if self.request.get("resource"):
      feeds = memcache.get(self.request.get("resource"))
      if feeds is not None:
        # good
        logging.debug("Memcache hit.")
      else:
        logging.debug("Memcache miss.")
        try:
          result = urlfetch.fetch(url=self.request.get("resource"), deadline=10)
          parser = LinkExtractor()
          parser.set_base_url(self.request.get("resource"))
          parser.feed(result.content)
          if parser.links:
            feeds = parser.links
          else:
            feeds = []

          if not feeds:
              # Let's check if by any chance this is actually not a feed?
              data = feedparser.parse(result.content)
              mimeType = "application/atom+xml"
              href = self.request.get("resource")
              if re.match("atom", data.version):
                  mimeType = "application/atom+xml"
              feeds = [{'title': data.feed.title, 'rel': 'self', 'type': mimeType, 'href': href}]

        except:
          feeds = []

        if not memcache.set(self.request.get("resource"), feeds, 86400):
          logging.error("Memcache set failed.")
        else:
          logging.debug("Memcache set.")

    self.response.out.write(template.render(os.path.join(os.path.dirname(__file__), 'templates', "subscribe.html"), {'appname': appname, 'feeds': feeds}))
Example #3
0
  def get(self):
    # We need to clean up the url first and remove any fragment
    site_url = urlparse.urldefrag(self.request.get("url"))[0]
    force = (self.request.get("force").lower()) in ['true', '1']
    extend = (self.request.get("extend").lower()) in ['true', '1']
    feeds = [] # default value

    if site_url:
      feeds = memcache.get(site_url + "." + str(extend))
      if feeds is not None and not force:
        # good
        logging.debug("Memcache hit.")
        self.render_json(feeds)
      else:
        logging.debug("Memcache miss.")
        try:
          result = urlfetch.fetch(url=site_url, deadline=10)
          parser = LinkExtractor()
          parser.set_base_url(site_url)
          parser.feed(result.content)
          if parser.links:
            feeds = parser.links
          else:
            feeds = []

          if not feeds:
            # Let's check if by any chance this is actually not a feed?
            data = feedparser.parse(result.content)
            if data.bozo == 0:
              feed = {'title': data.feed.get('title', ''), 'rel': 'self', 'type': 'application/atom+xml', 'href': site_url}
              links = data.feed.get('links', [])
              if extend:
                self.extend_feed(feed, links)
              feeds = [feed]
          else:
            if extend:
              for f in feeds:
                data = feedparser.parse(f['href'])
                links = data.feed.get('links', [])
                self.extend_feed(f, links)

        except:
          feeds = []

        if not memcache.set(site_url + "." + str(extend), feeds, 86400):
          logging.error("Memcache set failed.")
        else:
          logging.debug("Memcache set.")
        self.render_json(feeds)

    else:
      self.response.write(template.render(os.path.join(os.path.dirname(__file__), 'templates', "index.html"), {}))
Example #4
0
    def get(self):
        # We need to clean up the url first and remove any fragment
        site_url = urlparse.urldefrag(self.request.get("url"))[0]
        force = self.request.get("force")
        feeds = []  # default value
        if site_url:
            feeds = memcache.get(site_url)
            if feeds is not None and not force:
                # good
                logging.debug("Memcache hit.")
                self.render_json(feeds)
            else:
                logging.debug("Memcache miss.")
                try:
                    result = urlfetch.fetch(url=site_url, deadline=10)
                    parser = LinkExtractor()
                    parser.set_base_url(site_url)
                    parser.feed(result.content)
                    if parser.links:
                        feeds = parser.links
                    else:
                        feeds = []

                    if not feeds:
                        # Let's check if by any chance this is actually not a feed?
                        data = feedparser.parse(result.content)
                        mimeType = "application/atom+xml"
                        href = site_url
                        if re.match("atom", data.version):
                            mimeType = "application/atom+xml"
                        feeds = [{
                            'title': data.feed.title,
                            'rel': 'self',
                            'type': mimeType,
                            'href': href
                        }]

                except:
                    feeds = []

                if not memcache.set(site_url, feeds, 86400):
                    logging.error("Memcache set failed.")
                else:
                    logging.debug("Memcache set.")
                self.render_json(feeds)

        else:
            self.response.out.write(
                template.render(
                    os.path.join(os.path.dirname(__file__), 'templates',
                                 "index.html"), {}))