def expungeCache():
""" Expunge old entries from a cache of entries """
log = planet.logger
log.info("Determining feed subscriptions")
entry_count = {}
sources = config.cache_sources_directory()
for sub in config.subscriptions():
data=feedparser.parse(filename(sources,sub))
if not data.feed.has_key('id'): continue
if config.feed_options(sub).has_key('cache_keep_entries'):
entry_count[data.feed.id] = int(config.feed_options(sub)['cache_keep_entries'])
else:
entry_count[data.feed.id] = config.cache_keep_entries()
log.info("Listing cached entries")
cache = config.cache_directory()
dir=[(os.stat(file).st_mtime,file) for file in glob.glob(cache+"/*")
if not os.path.isdir(file)]
dir.sort()
dir.reverse()
for mtime,file in dir:
try:
entry=minidom.parse(file)
# determine source of entry
entry.normalize()
sources = entry.getElementsByTagName('source')
if not sources:
# no source determined, do not delete
log.debug("No source found for %s", file)
continue
ids = sources[0].getElementsByTagName('id')
if not ids:
# feed id not found, do not delete
log.debug("No source feed id found for %s", file)
continue
if ids[0].childNodes[0].nodeValue in entry_count:
# subscribed to feed, update entry count
entry_count[ids[0].childNodes[0].nodeValue] = entry_count[
ids[0].childNodes[0].nodeValue] - 1
if entry_count[ids[0].childNodes[0].nodeValue] >= 0:
# maximum not reached, do not delete
log.debug("Maximum not reached for %s from %s",
file, ids[0].childNodes[0].nodeValue)
continue
else:
# maximum reached
log.debug("Removing %s, maximum reached for %s",
file, ids[0].childNodes[0].nodeValue)
else:
# not subscribed
log.debug("Removing %s, not subscribed to %s",
file, ids[0].childNodes[0].nodeValue)
# remove old entry
os.unlink(file)
except:
log.error("Error parsing %s", file)
def spiderPlanet(only_if_new=False):
""" Spider (fetch) an entire planet """
log = planet.logger
global index
index = True
timeout = config.feed_timeout()
try:
socket.setdefaulttimeout(float(timeout))
log.info("Socket timeout set to %d seconds", timeout)
except:
try:
import timeoutsocket
timeoutsocket.setDefaultSocketTimeout(float(timeout))
log.info("Socket timeout set to %d seconds", timeout)
except:
log.warning("Timeout set to invalid value '%s', skipping", timeout)
from Queue import Queue
from threading import Thread
fetch_queue = Queue()
parse_queue = Queue()
threads = {}
http_cache = config.http_cache_directory()
# Should this be done in config?
if http_cache and not os.path.exists(http_cache):
os.makedirs(http_cache)
if int(config.spider_threads()):
# Start all the worker threads
for i in range(int(config.spider_threads())):
threads[i] = Thread(target=httpThread,
args=(i, fetch_queue, parse_queue, log))
threads[i].start()
else:
log.info("Building work queue")
# Load the fetch and parse work queues
for uri in config.subscriptions():
# read cached feed info
sources = config.cache_sources_directory()
feed_source = filename(sources, uri)
feed_info = feedparser.parse(feed_source)
if feed_info.feed and only_if_new:
log.info("Feed %s already in cache", uri)
continue
if feed_info.feed.get('planet_http_status', None) == '410':
log.info("Feed %s gone", uri)
continue
if threads and _is_http_uri(uri):
fetch_queue.put(item=(uri, feed_info))
else:
parse_queue.put(item=(uri, feed_info, uri))
# Mark the end of the fetch queue
for thread in threads.keys():
fetch_queue.put(item=(None, None))
# Process the results as they arrive
feeds_seen = {}
while fetch_queue.qsize() or parse_queue.qsize() or threads:
while parse_queue.qsize():
(uri, feed_info, feed) = parse_queue.get(False)
try:
if not hasattr(feed,
'headers') or int(feed.headers.status) < 300:
options = {}
if hasattr(feed_info, 'feed'):
options['etag'] = \
feed_info.feed.get('planet_http_etag',None)
try:
modified = time.strptime(
feed_info.feed.get('planet_http_last_modified',
None))
except:
pass
data = feedparser.parse(feed, **options)
else:
data = feedparser.FeedParserDict({
'version':
None,
'headers':
feed.headers,
'entries': [],
'feed': {},
'href':
feed.url,
'bozo':
0,
'status':
int(feed.headers.status)
})
# duplicate feed?
id = data.feed.get('id', None)
if not id: id = feed_info.feed.get('id', None)
href = uri
if data.has_key('href'): href = data.href
duplicate = None
if id and id in feeds_seen:
duplicate = id
elif href and href in feeds_seen:
duplicate = href
if duplicate:
feed_info.feed['planet_message'] = \
'duplicate subscription: ' + feeds_seen[duplicate]
log.warn('Duplicate subscription: %s and %s' %
(uri, feeds_seen[duplicate]))
if href: feed_info.feed['planet_http_location'] = href
if id: feeds_seen[id] = uri
if href: feeds_seen[href] = uri
# complete processing for the feed
writeCache(uri, feed_info, data)
except Exception, e:
import sys, traceback
type, value, tb = sys.exc_info()
log.error('Error processing %s', uri)
for line in (traceback.format_exception_only(type, value) +
traceback.format_tb(tb)):
log.error(line.rstrip())
time.sleep(0.1)
for index in threads.keys():
if not threads[index].isAlive():
del threads[index]
if not threads:
log.info("Finished threaded part of processing.")
def spiderPlanet(only_if_new = False):
""" Spider (fetch) an entire planet """
log = planet.logger
global index
index = True
timeout = config.feed_timeout()
try:
socket.setdefaulttimeout(float(timeout))
log.info("Socket timeout set to %d seconds", timeout)
except:
try:
import timeoutsocket
timeoutsocket.setDefaultSocketTimeout(float(timeout))
log.info("Socket timeout set to %d seconds", timeout)
except:
log.warning("Timeout set to invalid value '%s', skipping", timeout)
from Queue import Queue
from threading import Thread
fetch_queue = Queue()
parse_queue = Queue()
threads = {}
http_cache = config.http_cache_directory()
# Should this be done in config?
if http_cache and not os.path.exists(http_cache):
os.makedirs(http_cache)
if int(config.spider_threads()):
# Start all the worker threads
for i in range(int(config.spider_threads())):
threads[i] = Thread(target=httpThread,
args=(i,fetch_queue, parse_queue, log))
threads[i].start()
else:
log.info("Building work queue")
# Load the fetch and parse work queues
for uri in config.subscriptions():
# read cached feed info
sources = config.cache_sources_directory()
feed_source = filename(sources, uri)
feed_info = feedparser.parse(feed_source)
if feed_info.feed and only_if_new:
log.info("Feed %s already in cache", uri)
continue
if feed_info.feed.get('planet_http_status',None) == '410':
log.info("Feed %s gone", uri)
continue
if threads and _is_http_uri(uri):
fetch_queue.put(item=(uri, feed_info))
else:
parse_queue.put(item=(uri, feed_info, uri))
# Mark the end of the fetch queue
for thread in threads.keys():
fetch_queue.put(item=(None, None))
# Process the results as they arrive
feeds_seen = {}
while fetch_queue.qsize() or parse_queue.qsize() or threads:
while parse_queue.qsize():
(uri, feed_info, feed) = parse_queue.get(False)
try:
if not hasattr(feed,'headers') or int(feed.headers.status)<300:
options = {}
if hasattr(feed_info,'feed'):
options['etag'] = \
feed_info.feed.get('planet_http_etag',None)
try:
modified=time.strptime(
feed_info.feed.get('planet_http_last_modified',
None))
except:
pass
data = feedparser.parse(feed, **options)
else:
data = feedparser.FeedParserDict({'version': None,
'headers': feed.headers, 'entries': [], 'feed': {},
'href': feed.url, 'bozo': 0,
'status': int(feed.headers.status)})
# duplicate feed?
id = data.feed.get('id', None)
if not id: id = feed_info.feed.get('id', None)
href=uri
if data.has_key('href'): href=data.href
duplicate = None
if id and id in feeds_seen:
duplicate = id
elif href and href in feeds_seen:
duplicate = href
if duplicate:
feed_info.feed['planet_message'] = \
'duplicate subscription: ' + feeds_seen[duplicate]
log.warn('Duplicate subscription: %s and %s' %
(uri, feeds_seen[duplicate]))
if href: feed_info.feed['planet_http_location'] = href
if id: feeds_seen[id] = uri
if href: feeds_seen[href] = uri
# complete processing for the feed
writeCache(uri, feed_info, data)
except Exception, e:
import sys, traceback
type, value, tb = sys.exc_info()
log.error('Error processing %s', uri)
for line in (traceback.format_exception_only(type, value) +
traceback.format_tb(tb)):
log.error(line.rstrip())
time.sleep(0.1)
for index in threads.keys():
if not threads[index].isAlive():
del threads[index]
if not threads:
log.info("Finished threaded part of processing.")
def splice():
""" Splice together a planet from a cache of entries """
import planet
log = planet.logger
log.info("Loading cached data")
cache = config.cache_directory()
dir = [(os.stat(file).st_mtime, file) for file in glob.glob(cache + "/*")
if not os.path.isdir(file)]
dir.sort()
dir.reverse()
max_items = max([
config.items_per_page(templ)
for templ in config.template_files() or ['Planet']
])
doc = minidom.parseString('<feed xmlns="http://www.w3.org/2005/Atom"/>')
feed = doc.documentElement
# insert feed information
createTextElement(feed, 'title', config.name())
date(feed, 'updated', time.gmtime())
gen = createTextElement(feed, 'generator', config.generator())
gen.setAttribute('uri', config.generator_uri())
author = doc.createElement('author')
createTextElement(author, 'name', config.owner_name())
createTextElement(author, 'email', config.owner_email())
feed.appendChild(author)
if config.feed():
createTextElement(feed, 'id', config.feed())
link = doc.createElement('link')
link.setAttribute('rel', 'self')
link.setAttribute('href', config.feed())
if config.feedtype():
link.setAttribute('type', "application/%s+xml" % config.feedtype())
feed.appendChild(link)
if config.link():
link = doc.createElement('link')
link.setAttribute('rel', 'alternate')
link.setAttribute('href', config.link())
feed.appendChild(link)
# insert subscription information
sub_ids = []
feed.setAttribute('xmlns:planet', planet.xmlns)
sources = config.cache_sources_directory()
for sub in config.subscriptions():
data = feedparser.parse(filename(sources, sub))
if data.feed.has_key('id'): sub_ids.append(data.feed.id)
if not data.feed: continue
xdoc = minidom.parseString('''<planet:source xmlns:planet="%s"
xmlns="http://www.w3.org/2005/Atom"/>\n''' % planet.xmlns)
reconstitute.source(xdoc.documentElement, data.feed, None, None)
feed.appendChild(xdoc.documentElement)
index = idindex.open()
# insert entry information
items = 0
count = {}
new_feed_items = config.new_feed_items()
for mtime, file in dir:
if index != None:
base = os.path.basename(file)
if index.has_key(base) and index[base] not in sub_ids: continue
try:
entry = minidom.parse(file)
# verify that this entry is currently subscribed to and that the
# number of entries contributed by this feed does not exceed
# config.new_feed_items
entry.normalize()
sources = entry.getElementsByTagName('source')
if sources:
ids = sources[0].getElementsByTagName('id')
if ids:
id = ids[0].childNodes[0].nodeValue
count[id] = count.get(id, 0) + 1
if new_feed_items and count[id] > new_feed_items: continue
if id not in sub_ids:
ids = sources[0].getElementsByTagName('planet:id')
if not ids: continue
id = ids[0].childNodes[0].nodeValue
if id not in sub_ids: continue
# add entry to feed
feed.appendChild(entry.documentElement)
items = items + 1
if items >= max_items: break
except:
log.error("Error parsing %s", file)
if index: index.close()
return doc
def splice():
""" Splice together a planet from a cache of entries """
import planet
log = planet.logger
log.info("Loading cached data")
cache = config.cache_directory()
dir=[(os.stat(file).st_mtime,file) for file in glob.glob(cache+"/*")
if not os.path.isdir(file)]
dir.sort()
dir.reverse()
max_items=max([config.items_per_page(templ)
for templ in config.template_files() or ['Planet']])
doc = minidom.parseString('<feed xmlns="http://www.w3.org/2005/Atom"/>')
feed = doc.documentElement
# insert feed information
createTextElement(feed, 'title', config.name())
date(feed, 'updated', time.gmtime())
gen = createTextElement(feed, 'generator', config.generator())
gen.setAttribute('uri', config.generator_uri())
author = doc.createElement('author')
createTextElement(author, 'name', config.owner_name())
createTextElement(author, 'email', config.owner_email())
feed.appendChild(author)
if config.feed():
createTextElement(feed, 'id', config.feed())
link = doc.createElement('link')
link.setAttribute('rel', 'self')
link.setAttribute('href', config.feed())
if config.feedtype():
link.setAttribute('type', "application/%s+xml" % config.feedtype())
feed.appendChild(link)
if config.pubsubhubbub_hub():
hub = doc.createElement('link')
hub.setAttribute('rel', 'hub')
hub.setAttribute('href', config.pubsubhubbub_hub())
feed.appendChild(hub)
if config.link():
link = doc.createElement('link')
link.setAttribute('rel', 'alternate')
link.setAttribute('href', config.link())
feed.appendChild(link)
# insert subscription information
sub_ids = []
feed.setAttribute('xmlns:planet',planet.xmlns)
sources = config.cache_sources_directory()
for sub in config.subscriptions():
data=feedparser.parse(filename(sources,sub))
if data.feed.has_key('id'): sub_ids.append(data.feed.id)
if not data.feed: continue
# warn on missing links
if not data.feed.has_key('planet_message'):
if not data.feed.has_key('links'): data.feed['links'] = []
for link in data.feed.links:
if link.rel == 'self': break
else:
log.debug('missing self link for ' + sub)
for link in data.feed.links:
if link.rel == 'alternate' and 'html' in link.type: break
else:
log.debug('missing html link for ' + sub)
xdoc=minidom.parseString('''<planet:source xmlns:planet="%s"
xmlns="http://www.w3.org/2005/Atom"/>\n''' % planet.xmlns)
reconstitute.source(xdoc.documentElement, data.feed, None, None)
feed.appendChild(xdoc.documentElement)
index = idindex.open()
# insert entry information
items = 0
count = {}
atomNS='http://www.w3.org/2005/Atom'
new_feed_items = config.new_feed_items()
for mtime,file in dir:
if index != None:
base = os.path.basename(file)
if index.has_key(base) and index[base] not in sub_ids: continue
try:
entry=minidom.parse(file)
# verify that this entry is currently subscribed to and that the
# number of entries contributed by this feed does not exceed
# config.new_feed_items
entry.normalize()
sources = entry.getElementsByTagNameNS(atomNS, 'source')
if sources:
ids = sources[0].getElementsByTagName('id')
if ids:
id = ids[0].childNodes[0].nodeValue
count[id] = count.get(id,0) + 1
if new_feed_items and count[id] > new_feed_items: continue
if id not in sub_ids:
ids = sources[0].getElementsByTagName('planet:id')
if not ids: continue
id = ids[0].childNodes[0].nodeValue
if id not in sub_ids:
log.warn('Skipping: ' + id)
if id not in sub_ids: continue
# add entry to feed
feed.appendChild(entry.documentElement)
items = items + 1
if items >= max_items: break
except:
log.error("Error parsing %s", file)
if index: index.close()
return doc
def expungeCache():
""" Expunge old entries from a cache of entries """
log = planet.logger
log.info("Determining feed subscriptions")
entry_count = {}
sources = config.cache_sources_directory()
for sub in config.subscriptions():
data = feedparser.parse(filename(sources, sub))
if not data.feed.has_key('id'): continue
if config.feed_options(sub).has_key('cache_keep_entries'):
entry_count[data.feed.id] = int(
config.feed_options(sub)['cache_keep_entries'])
else:
entry_count[data.feed.id] = config.cache_keep_entries()
log.info("Listing cached entries")
cache = config.cache_directory()
dir = [(os.stat(file).st_mtime, file) for file in glob.glob(cache + "/*")
if not os.path.isdir(file)]
dir.sort()
dir.reverse()
for mtime, file in dir:
try:
entry = minidom.parse(file)
# determine source of entry
entry.normalize()
sources = entry.getElementsByTagName('source')
if not sources:
# no source determined, do not delete
log.debug("No source found for %s", file)
continue
ids = sources[0].getElementsByTagName('id')
if not ids:
# feed id not found, do not delete
log.debug("No source feed id found for %s", file)
continue
if ids[0].childNodes[0].nodeValue in entry_count:
# subscribed to feed, update entry count
entry_count[ids[0].childNodes[0].nodeValue] = entry_count[
ids[0].childNodes[0].nodeValue] - 1
if entry_count[ids[0].childNodes[0].nodeValue] >= 0:
# maximum not reached, do not delete
log.debug("Maximum not reached for %s from %s", file,
ids[0].childNodes[0].nodeValue)
continue
else:
# maximum reached
log.debug("Removing %s, maximum reached for %s", file,
ids[0].childNodes[0].nodeValue)
else:
# not subscribed
log.debug("Removing %s, not subscribed to %s", file,
ids[0].childNodes[0].nodeValue)
# remove old entry
os.unlink(file)
except:
log.error("Error parsing %s", file)
def splice():
""" Splice together a planet from a cache of entries """
import planet
log = planet.logger
log.info("Loading cached data")
cache = config.cache_directory()
dir = [(os.stat(file).st_mtime, file) for file in glob.glob(cache + "/*")
if not os.path.isdir(file)]
dir.sort()
dir.reverse()
max_items = max([
config.items_per_page(templ)
for templ in config.template_files() or ['Planet']
])
doc = minidom.parseString('<feed xmlns="http://www.w3.org/2005/Atom"/>')
feed = doc.documentElement
# insert feed information
createTextElement(feed, 'title', config.name())
date(feed, 'updated', time.gmtime())
gen = createTextElement(feed, 'generator', config.generator())
gen.setAttribute('uri', config.generator_uri())
author = doc.createElement('author')
createTextElement(author, 'name', config.owner_name())
createTextElement(author, 'email', config.owner_email())
feed.appendChild(author)
if config.feed():
createTextElement(feed, 'id', config.feed())
link = doc.createElement('link')
link.setAttribute('rel', 'self')
link.setAttribute('href', config.feed())
if config.feedtype():
link.setAttribute('type', "application/%s+xml" % config.feedtype())
feed.appendChild(link)
if config.pubsubhubbub_hub():
hub = doc.createElement('link')
hub.setAttribute('rel', 'hub')
hub.setAttribute('href', config.pubsubhubbub_hub())
feed.appendChild(hub)
if config.link():
link = doc.createElement('link')
link.setAttribute('rel', 'alternate')
link.setAttribute('href', config.link())
feed.appendChild(link)
# insert subscription information
sub_ids = []
feed.setAttribute('xmlns:planet', planet.xmlns)
sources = config.cache_sources_directory()
for sub in config.subscriptions():
data = feedparser.parse(filename(sources, sub))
if data.feed.has_key('id'):
sub_ids.append(data.feed.id)
if not data.feed:
continue
# warn on missing links
if not data.feed.has_key('planet_message'):
if not data.feed.has_key('links'):
data.feed['links'] = []
for link in data.feed.links:
if link.rel == 'self':
break
else:
log.debug('missing self link for ' + sub)
for link in data.feed.links:
if link.rel == 'alternate' and 'html' in link.type:
break
else:
log.debug('missing html link for ' + sub)
xdoc = minidom.parseString('''<planet:source xmlns:planet="%s"
xmlns="http://www.w3.org/2005/Atom"/>\n''' % planet.xmlns)
reconstitute.source(xdoc.documentElement, data.feed, None, None)
feed.appendChild(xdoc.documentElement)
index = idindex.open()
# insert entry information
items = 0
count = {}
atomNS = 'http://www.w3.org/2005/Atom'
new_feed_items = config.new_feed_items()
posted_urls = set()
if config.post_to_twitter():
if os.path.exists(posted_urls_file):
try:
with open(posted_urls_file, 'rb') as f:
posted_urls = pickle.load(f)
except Exception as ex:
log.error("Error reading posted_urls %s", ex)
# print(posted_urls)
for mtime, file in dir:
if index != None:
base = os.path.basename(file)
if index.has_key(base) and index[base] not in sub_ids:
continue
try:
entry = minidom.parse(file)
# verify that this entry is currently subscribed to and that the
# number of entries contributed by this feed does not exceed
# config.new_feed_items
entry.normalize()
sources = entry.getElementsByTagNameNS(atomNS, 'source')
if sources:
ids = sources[0].getElementsByTagName('id')
if ids:
id = ids[0].childNodes[0].nodeValue
count[id] = count.get(id, 0) + 1
if new_feed_items and count[id] > new_feed_items:
continue
if id not in sub_ids:
ids = sources[0].getElementsByTagName('planet:id')
if not ids:
continue
id = ids[0].childNodes[0].nodeValue
if id not in sub_ids:
log.warn('Skipping: ' + id)
if id not in sub_ids:
continue
# Twitter integration
if config.post_to_twitter():
url = None
twitter = None
title = "Untitled post..."
links = entry.getElementsByTagName('link')
if links:
for link in links:
if link.hasAttribute('rel') and link.hasAttribute(
'type') and link.hasAttribute('href'):
if (link.getAttribute('rel') == 'alternate' and
link.getAttribute('type') == 'text/html'):
url = link.getAttribute('href')
break
titles = entry.getElementsByTagName('title')
if titles:
title = unicode(
titles[0].firstChild.nodeValue.encode('utf-8'),
'utf-8').strip()
handles = entry.getElementsByTagName('planet:twitter')
if (handles):
twitter = unicode(
handles[0].firstChild.nodeValue.encode('utf-8'),
"utf-8")
if url is not None and url not in posted_urls:
# log.debug("Going to post URL to Twitter: twitter='{}' title='{}', url='{}'".format(twitter, title, url))
txt_append = u''
if twitter:
txt_append = u" (by @" + twitter.encode(
'utf-8').strip() + u")"
max_title_len = 280 - 20 - len(txt_append)
if (len(title) > max_title_len):
title = title[:max_title_len]
txt = title + txt_append + u"\n" + url
log.debug(u"Text to post '{}'".format(txt))
try:
posted_urls.add(url)
config.twitter_api.update_status(txt)
except Exception as ex:
log.error(u"Error posting to Twitter: %s", ex)
# add entry to feed
feed.appendChild(entry.documentElement)
items = items + 1
if items >= max_items:
break
except Exception as ex:
log.error("Error parsing %s: %s", file, ex)
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_traceback,
limit=2,
file=sys.stdout)
if config.post_to_twitter():
with open(posted_urls_file, 'wb') as f:
pickle.dump(posted_urls, f, protocol=pickle.HIGHEST_PROTOCOL)
if index:
index.close()
return doc
def splice():
""" Splice together a planet from a cache of entries """
import planet
log = planet.getLogger(config.log_level(),config.log_format())
log.info("Loading cached data")
cache = config.cache_directory()
dir=[(os.stat(file).st_mtime,file) for file in glob.glob(cache+"/*")
if not os.path.isdir(file)]
dir.sort()
dir.reverse()
max_items=max([config.items_per_page(templ)
for templ in config.template_files() or ['Planet']])
doc = minidom.parseString('<feed xmlns="http://www.w3.org/2005/Atom"/>')
feed = doc.documentElement
# insert feed information
createTextElement(feed, 'title', config.name())
date(feed, 'updated', time.gmtime())
gen = createTextElement(feed, 'generator', config.generator())
gen.setAttribute('uri', config.generator_uri())
author = doc.createElement('author')
createTextElement(author, 'name', config.owner_name())
createTextElement(author, 'email', config.owner_email())
feed.appendChild(author)
if config.feed():
createTextElement(feed, 'id', config.feed())
link = doc.createElement('link')
link.setAttribute('rel', 'self')
link.setAttribute('href', config.feed())
if config.feedtype():
link.setAttribute('type', "application/%s+xml" % config.feedtype())
feed.appendChild(link)
if config.link():
link = doc.createElement('link')
link.setAttribute('rel', 'alternate')
link.setAttribute('href', config.link())
feed.appendChild(link)
# insert subscription information
sub_ids = []
feed.setAttribute('xmlns:planet',planet.xmlns)
sources = config.cache_sources_directory()
for sub in config.subscriptions():
data=feedparser.parse(filename(sources,sub))
if data.feed.has_key('id'): sub_ids.append(data.feed.id)
if not data.feed: continue
xdoc=minidom.parseString('''<planet:source xmlns:planet="%s"
xmlns="http://www.w3.org/2005/Atom"/>\n''' % planet.xmlns)
reconstitute.source(xdoc.documentElement, data.feed, None, None)
feed.appendChild(xdoc.documentElement)
index = idindex.open()
# insert entry information
items = 0
for mtime,file in dir:
if index != None:
base = os.path.basename(file)
if index.has_key(base) and index[base] not in sub_ids: continue
try:
entry=minidom.parse(file)
# verify that this entry is currently subscribed to
entry.normalize()
sources = entry.getElementsByTagName('source')
if sources:
ids = sources[0].getElementsByTagName('id')
if ids and ids[0].childNodes[0].nodeValue not in sub_ids:
ids = sources[0].getElementsByTagName('planet:id')
if not ids: continue
if ids[0].childNodes[0].nodeValue not in sub_ids: continue
# add entry to feed
feed.appendChild(entry.documentElement)
items = items + 1
if items >= max_items: break
except:
log.error("Error parsing %s", file)
if index: index.close()
return doc
