try:
spider.spiderPlanet(only_if_new=only_if_new)
except Exception, e:
print e
from planet import splice
doc = splice.splice()
if debug_splice:
from planet import logger
logger.info('writing debug.atom')
debug=open('debug.atom','w')
try:
from lxml import etree
from StringIO import StringIO
tree = etree.tostring(etree.parse(StringIO(doc.toxml())))
debug.write(etree.tostring(tree, pretty_print=True))
except:
debug.write(doc.toprettyxml(indent=' ', encoding='utf-8'))
debug.close
splice.apply(doc.toxml('utf-8'))
if config.pubsubhubbub_hub() and not no_publish:
from planet import publish
publish.publish(config)
if expunge:
from planet import expunge
expunge.expungeCache
try:
spider.spiderPlanet(only_if_new=only_if_new)
except Exception, e:
print e
from planet import splice
doc = splice.splice()
if debug_splice:
from planet import logger
logger.info('writing debug.atom')
debug = open('debug.atom', 'w')
try:
from lxml import etree
from StringIO import StringIO
tree = etree.tostring(etree.parse(StringIO(doc.toxml())))
debug.write(etree.tostring(tree, pretty_print=True))
except:
debug.write(doc.toprettyxml(indent=' ', encoding='utf-8'))
debug.close
splice.apply(doc.toxml('utf-8'))
if config.pubsubhubbub_hub() and not no_publish:
from planet import publish
publish.publish(config)
if expunge:
from planet import expunge
expunge.expungeCache
def test_pubsubhubbub_hub(self):
self.assertEqual("http://pubsubhubbub.appspot.com", config.pubsubhubbub_hub())
def template_info(source):
""" get template information from a feedparser output """
# wire in support for planet:source, call feedparser, unplug planet:source
mixin=feedparser._FeedParserMixin
mixin._start_planet_source = mixin._start_source
mixin._end_planet_source = \
new.instancemethod(_end_planet_source, None, mixin)
data=feedparser.parse(source)
del mixin._start_planet_source
del mixin._end_planet_source
# apply rules to convert feed parser output to htmltmpl input
output = {'Channels': [], 'Items': []}
output.update(tmpl_mapper(data.feed, Base))
sources = []
for feed in data.feed.get('sources',[]):
source = tmpl_mapper(feed, Base)
sources.append([source.get('name'), source])
sources.sort()
output['Channels'] = [source for name,source in sources]
for entry in data.entries:
output['Items'].append(tmpl_mapper(entry, Items))
# synthesize isPermaLink attribute
for item in output['Items']:
if item.get('id') == item.get('link'):
item['guid_isPermaLink']='true'
else:
item['guid_isPermaLink']='false'
# feed level information
output['generator'] = config.generator_uri()
output['name'] = config.name()
output['link'] = config.link()
output['owner_name'] = config.owner_name()
output['owner_email'] = config.owner_email()
output['pubsubhubbub_hub'] = config.pubsubhubbub_hub()
if config.feed():
output['feed'] = config.feed()
output['feedtype'] = config.feed().find('rss')>=0 and 'rss' or 'atom'
# date/time information
date = time.gmtime()
output['date'] = PlanetDate(date)
output['date_iso'] = Rfc3399(date)
output['date_822'] = Rfc822(date)
# remove new_dates and new_channels that aren't "new"
date = channel = None
for item in output['Items']:
if item.has_key('new_date'):
if item['new_date'] == date:
del item['new_date']
else:
date = item['new_date']
if item.has_key('new_channel'):
if item['new_channel'] == channel and not item.has_key('new_date'):
del item['new_channel']
else:
channel = item['new_channel']
return output
def template_info(source):
""" get template information from a feedparser output """
# wire in support for planet:source, call feedparser, unplug planet:source
mixin=feedparser._FeedParserMixin
mixin._start_planet_source = mixin._start_source
mixin._end_planet_source = \
new.instancemethod(_end_planet_source, None, mixin)
data=feedparser.parse(source)
del mixin._start_planet_source
del mixin._end_planet_source
# apply rules to convert feed parser output to htmltmpl input
output = {'Channels': [], 'Items': []}
output.update(tmpl_mapper(data.feed, Base))
sources = []
for feed in data.feed.get('sources',[]):
source = tmpl_mapper(feed, Base)
sources.append([source.get('name'), source])
sources.sort(lambda x,y: cmp(x[0].upper(), y[0].upper()))
output['Channels'] = [source for name,source in sources]
for entry in data.entries:
output['Items'].append(tmpl_mapper(entry, Items))
# synthesize isPermaLink attribute
for item in output['Items']:
if item.get('id') == item.get('link'):
item['guid_isPermaLink']='true'
else:
item['guid_isPermaLink']='false'
# feed level information
output['generator'] = config.generator_uri()
output['name'] = config.name()
output['link'] = config.link()
output['owner_name'] = config.owner_name()
output['owner_email'] = config.owner_email()
output['pubsubhubbub_hub'] = config.pubsubhubbub_hub()
if config.feed():
output['feed'] = config.feed()
output['feedtype'] = config.feed().find('rss')>=0 and 'rss' or 'atom'
# date/time information
date = time.gmtime()
output['date'] = PlanetDate(date)
output['date_iso'] = Rfc3399(date)
output['date_822'] = Rfc822(date)
# remove new_dates and new_channels that aren't "new"
date = channel = None
for item in output['Items']:
if item.has_key('new_date'):
if item['new_date'] == date:
del item['new_date']
else:
date = item['new_date']
if item.has_key('new_channel'):
if item['new_channel'] == channel and not item.has_key('new_date'):
del item['new_channel']
else:
channel = item['new_channel']
return output
def test_pubsubhubbub_hub(self):
self.assertEqual('http://pubsubhubbub.appspot.com', config.pubsubhubbub_hub())
