def get(self):
# We need to clean up the url first and remove any fragment
site_url = urlparse.urldefrag(self.request.get("url"))[0]
force = (self.request.get("force").lower()) in ['true', '1']
extend = (self.request.get("extend").lower()) in ['true', '1']
feeds = [] # default value
if site_url:
feeds = memcache.get(site_url + "." + str(extend))
if feeds is not None and not force:
# good
logging.debug("Memcache hit.")
self.render_json(feeds)
else:
logging.debug("Memcache miss.")
try:
result = urlfetch.fetch(url=site_url, deadline=10)
parser = LinkExtractor()
parser.set_base_url(site_url)
parser.feed(result.content)
if parser.links:
feeds = parser.links
else:
feeds = []
if not feeds:
# Let's check if by any chance this is actually not a feed?
data = feedparser.parse(result.content)
if data.bozo == 0:
feed = {'title': data.feed.get('title', ''), 'rel': 'self', 'type': 'application/atom+xml', 'href': site_url}
links = data.feed.get('links', [])
if extend:
self.extend_feed(feed, links)
feeds = [feed]
else:
if extend:
for f in feeds:
data = feedparser.parse(f['href'])
links = data.feed.get('links', [])
self.extend_feed(f, links)
except:
feeds = []
if not memcache.set(site_url + "." + str(extend), feeds, 86400):
logging.error("Memcache set failed.")
else:
logging.debug("Memcache set.")
self.render_json(feeds)
else:
self.response.write(template.render(os.path.join(os.path.dirname(__file__), 'templates', "index.html"), {}))
def get(self):
# We need to clean up the url first and remove any fragment
site_url = urlparse.urldefrag(self.request.get("url"))[0]
force = self.request.get("force")
feeds = [] # default value
if site_url:
feeds = memcache.get(site_url)
if feeds is not None and not force:
# good
logging.debug("Memcache hit.")
self.render_json(feeds)
else:
logging.debug("Memcache miss.")
try:
result = urlfetch.fetch(url=site_url, deadline=10)
parser = LinkExtractor()
parser.set_base_url(site_url)
parser.feed(result.content)
if parser.links:
feeds = parser.links
else:
feeds = []
if not feeds:
# Let's check if by any chance this is actually not a feed?
data = feedparser.parse(result.content)
mimeType = "application/atom+xml"
href = site_url
if re.match("atom", data.version):
mimeType = "application/atom+xml"
feeds = [{
'title': data.feed.title,
'rel': 'self',
'type': mimeType,
'href': href
}]
except:
feeds = []
if not memcache.set(site_url, feeds, 86400):
logging.error("Memcache set failed.")
else:
logging.debug("Memcache set.")
self.render_json(feeds)
else:
self.response.out.write(
template.render(
os.path.join(os.path.dirname(__file__), 'templates',
"index.html"), {}))
def get(self):
# We need to clean up the url first and remove any fragment
site_url = urlparse.urldefrag(self.request.get("url"))[0]
force = self.request.get("force")
feeds = [] # default value
if site_url:
feeds = memcache.get(site_url)
if feeds is not None and not force:
# good
logging.debug("Memcache hit.")
self.render_json(feeds)
else:
logging.debug("Memcache miss.")
try:
result = urlfetch.fetch(url=site_url, deadline=10)
parser = LinkExtractor()
parser.set_base_url(site_url)
parser.feed(result.content)
if parser.links:
feeds = parser.links
else:
feeds = []
if not feeds:
# Let's check if by any chance this is actually not a feed?
data = feedparser.parse(result.content)
mimeType = "application/atom+xml"
href = site_url
if re.match("atom", data.version):
mimeType = "application/atom+xml"
feeds = [{'title': data.feed.title, 'rel': 'self', 'type': mimeType, 'href': href}]
except:
feeds = []
if not memcache.set(site_url, feeds, 86400):
logging.error("Memcache set failed.")
else:
logging.debug("Memcache set.")
self.render_json(feeds)
else:
self.response.out.write(template.render(os.path.join(os.path.dirname(__file__), 'templates', "index.html"), {}))
def get(self):
feeds = []
if self.request.get("resource"):
feeds = memcache.get(self.request.get("resource"))
if feeds is not None:
# good
logging.debug("Memcache hit.")
else:
logging.debug("Memcache miss.")
try:
result = urlfetch.fetch(url=self.request.get("resource"), deadline=10)
parser = LinkExtractor()
parser.set_base_url(self.request.get("resource"))
parser.feed(result.content)
if parser.links:
feeds = parser.links
else:
feeds = []
if not feeds:
# Let's check if by any chance this is actually not a feed?
data = feedparser.parse(result.content)
mimeType = "application/atom+xml"
href = self.request.get("resource")
if re.match("atom", data.version):
mimeType = "application/atom+xml"
feeds = [{'title': data.feed.title, 'rel': 'self', 'type': mimeType, 'href': href}]
except:
feeds = []
if not memcache.set(self.request.get("resource"), feeds, 86400):
logging.error("Memcache set failed.")
else:
logging.debug("Memcache set.")
self.response.out.write(template.render(os.path.join(os.path.dirname(__file__), 'templates', "subscribe.html"), {'appname': appname, 'feeds': feeds}))