def test_feed_options(self):
feeds = dict([(split(feed)[1],feed) for feed in config.subscriptions()])
feed1 = feeds['testfeed1a.atom']
self.assertEqual('one', config.feed_options(feed1)['name'])
feed2 = feeds['testfeed2.atom']
self.assertEqual('two', config.feed_options(feed2)['name'])
def test_feed_options(self):
feeds = dict([(split(feed)[1], feed) for feed in config.subscriptions()])
feed1 = feeds['testfeed1a.atom']
self.assertEqual('one', config.feed_options(feed1)['name'])
feed2 = feeds['testfeed2.atom']
self.assertEqual('two', config.feed_options(feed2)['name'])
def test_feeds(self):
feeds = [split(feed)[1] for feed in config.subscriptions()]
feeds.sort()
self.assertEqual([
'testfeed0.atom', 'testfeed1a.atom', 'testfeed2.atom',
'testfeed3.rss'
], feeds)
def test_online_accounts(self):
config.load('tests/data/config/foaf.ini')
feeds = config.subscriptions()
feeds.sort()
self.assertEqual(['http://api.flickr.com/services/feeds/' +
'photos_public.gne?id=77366516@N00',
'http://del.icio.us/rss/eliast',
'http://torrez.us/feed/rdf'], feeds)
def test_online_accounts(self):
config.load('tests/data/config/foaf.ini')
feeds = config.subscriptions()
feeds.sort()
self.assertEqual([
'http://api.flickr.com/services/feeds/' +
'photos_public.gne?id=77366516@N00',
'http://del.icio.us/rss/eliast', 'http://torrez.us/feed/rdf'
], feeds)
def test_recursive(self):
config.load('tests/data/config/foaf-deep.ini')
feeds = config.subscriptions()
feeds.sort()
self.assertEqual(['http://api.flickr.com/services/feeds/photos_public.gne?id=77366516@N00',
'http://del.icio.us/rss/eliast', 'http://del.icio.us/rss/leef',
'http://del.icio.us/rss/rubys', 'http://intertwingly.net/blog/atom.xml',
'http://thefigtrees.net/lee/life/atom.xml',
'http://torrez.us/feed/rdf'], feeds)
def test_recursive(self):
config.load('tests/data/config/foaf-deep.ini')
feeds = config.subscriptions()
feeds.sort()
self.assertEqual([
'http://api.flickr.com/services/feeds/photos_public.gne?id=77366516@N00',
'http://del.icio.us/rss/eliast', 'http://del.icio.us/rss/leef',
'http://del.icio.us/rss/rubys',
'http://intertwingly.net/blog/atom.xml',
'http://thefigtrees.net/lee/life/atom.xml',
'http://torrez.us/feed/rdf'
], feeds)
def test_multiple_subscriptions(self):
config.load('tests/data/config/foaf-multiple.ini')
self.assertEqual(2, len(config.reading_lists()))
feeds = config.subscriptions()
feeds.sort()
self.assertEqual(5, len(feeds))
self.assertEqual([
'http://api.flickr.com/services/feeds/' +
'photos_public.gne?id=77366516@N00',
'http://api.flickr.com/services/feeds/' +
'photos_public.gne?id=SOMEID', 'http://del.icio.us/rss/SOMEID',
'http://del.icio.us/rss/eliast', 'http://torrez.us/feed/rdf'
], feeds)
def test_multiple_subscriptions(self):
config.load('tests/data/config/foaf-multiple.ini')
self.assertEqual(2,len(config.reading_lists()))
feeds = config.subscriptions()
feeds.sort()
self.assertEqual(5,len(feeds))
self.assertEqual(['http://api.flickr.com/services/feeds/' +
'photos_public.gne?id=77366516@N00',
'http://api.flickr.com/services/feeds/' +
'photos_public.gne?id=SOMEID',
'http://del.icio.us/rss/SOMEID',
'http://del.icio.us/rss/eliast',
'http://torrez.us/feed/rdf'], feeds)
def test_feeds(self):
feeds = config.subscriptions()
feeds.sort()
self.assertEqual(['feed1', 'feed2'], feeds)
def run(script, doc, output_file=None, options={}):
""" process an Genshi template """
context = Context(**options)
tmpl_fileobj = open(script)
tmpl = MarkupTemplate(tmpl_fileobj, script)
tmpl_fileobj.close()
if not output_file:
# filter
context.push({'input':XMLParser(StringIO(doc))})
else:
# template
import time
from planet import config,feedparser
from planet.spider import filename
# gather a list of subscriptions, feeds
global subscriptions
feeds = []
sources = config.cache_sources_directory()
for sub in config.subscriptions():
data=feedparser.parse(filename(sources,sub))
data.feed.config = norm(dict(config.parser.items(sub)))
if data.feed.has_key('link'):
feeds.append((data.feed.config.get('name',''),data.feed))
subscriptions.append(norm(sub))
feeds.sort()
# annotate each entry
new_date_format = config.new_date_format()
vars = feedparser.parse(StringIO(doc))
vars.feeds = [value for name,value in feeds]
last_feed = None
last_date = None
for entry in vars.entries:
entry.source.config = find_config(config, entry.source)
# add new_feed and new_date fields
if 'id' in entry.source:
entry.new_feed = entry.source.id
else:
entry.new_feed = None
entry.new_date = date = None
if entry.has_key('published_parsed'): date=entry.published_parsed
if entry.has_key('updated_parsed'): date=entry.updated_parsed
if date: entry.new_date = time.strftime(new_date_format, date)
# remove new_feed and new_date fields if not "new"
if entry.new_date == last_date:
entry.new_date = None
if entry.new_feed == last_feed:
entry.new_feed = None
else:
last_feed = entry.new_feed
elif entry.new_date:
last_date = entry.new_date
last_feed = None
# add streams for all text constructs
for key in entry.keys():
if key.endswith("_detail") and entry[key].has_key('type') and \
entry[key].has_key('value'):
streamify(entry[key],entry.source.planet_bozo)
if entry.has_key('content'):
for content in entry.content:
streamify(content,entry.source.planet_bozo)
# add cumulative feed information to the Genshi context
vars.feed.config = dict(config.parser.items('Planet',True))
context.push(vars)
# apply template
output=tmpl.generate(context).render('xml')
if output_file:
out_file = open(output_file,'w')
out_file.write(output)
out_file.close()
else:
return output
def test_feeds(self):
feeds = config.subscriptions()
feeds.sort()
self.assertEqual(['feed1', 'feed2'], feeds)
def test_feeds(self):
feeds = [split(feed)[1] for feed in config.subscriptions()]
feeds.sort()
self.assertEqual(['testfeed0.atom', 'testfeed1a.atom',
'testfeed2.atom', 'testfeed3.rss'], feeds)
def test_feeds(self):
feeds = config.subscriptions()
feeds.sort()
self.assertEqual(["feed1", "feed2"], feeds)
from ConfigParser import ConfigParser
# load config files (default: config.ini)
for arg in sys.argv[1:]:
config.load(arg)
if len(sys.argv) == 1:
config.load('config.ini')
from Queue import Queue
from threading import Thread
# determine which subscriptions have no icon but do have a html page
fetch_queue = Queue()
html = ['text/html', 'application/xhtml+xml']
sources = config.cache_sources_directory()
for sub in config.subscriptions():
data = feedparser.parse(filename(sources, sub))
if data.feed.get('icon'): continue
if not data.feed.get('links'): continue
for link in data.feed.links:
if link.rel == 'alternate' and link.type in html:
fetch_queue.put((sub, link.href))
break
# find the favicon for a given webpage
def favicon(page):
parser = html5parser.HTMLParser(tree=treebuilders.getTreeBuilder('dom'))
doc = parser.parse(urlopen(page))
favicon = urljoin(page, '/favicon.ico')
for link in doc.getElementsByTagName('link'):
def run(script, doc, output_file=None, options={}):
""" process an Genshi template """
context = Context(**options)
tmpl_fileobj = open(script)
tmpl = MarkupTemplate(tmpl_fileobj, script)
tmpl_fileobj.close()
if not output_file:
# filter
context.push({'input':XMLParser(StringIO(doc))})
else:
# template
import time
from planet import config,feedparser
from planet.spider import filename
# gather a list of subscriptions, feeds
global subscriptions
feeds = []
sources = config.cache_sources_directory()
for sub in config.subscriptions():
data=feedparser.parse(filename(sources,sub))
data.feed.config = norm(dict(config.parser.items(sub)))
if data.feed.has_key('link'):
feeds.append((data.feed.config.get('name',''),data.feed))
subscriptions.append(norm(sub))
feeds.sort()
# annotate each entry
new_date_format = config.new_date_format()
vars = feedparser.parse(StringIO(doc))
vars.feeds = [value for name,value in feeds]
last_feed = None
last_date = None
for entry in vars.entries:
entry.source.config = find_config(config, entry.source)
# add new_feed and new_date fields
entry.new_feed = entry.source.id
entry.new_date = date = None
if entry.has_key('published_parsed'): date=entry.published_parsed
if entry.has_key('updated_parsed'): date=entry.updated_parsed
if date: entry.new_date = time.strftime(new_date_format, date)
# remove new_feed and new_date fields if not "new"
if entry.new_date == last_date:
entry.new_date = None
if entry.new_feed == last_feed:
entry.new_feed = None
else:
last_feed = entry.new_feed
elif entry.new_date:
last_date = entry.new_date
last_feed = None
# add streams for all text constructs
for key in entry.keys():
if key.endswith("_detail") and entry[key].has_key('type') and \
entry[key].has_key('value'):
streamify(entry[key],entry.source.planet_bozo)
if entry.has_key('content'):
for content in entry.content:
streamify(content,entry.source.planet_bozo)
# add cumulative feed information to the Genshi context
vars.feed.config = dict(config.parser.items('Planet',True))
context.push(vars)
# apply template
output=tmpl.generate(context).render('xml')
if output_file:
out_file = open(output_file,'w')
out_file.write(output)
out_file.close()
else:
return output
