def writeCache(feed_uri, feed_info, data):
log = planet.logger
sources = config.cache_sources_directory()
# capture http status
if not data.has_key("status"):
if data.has_key("entries") and len(data.entries) > 0:
data.status = 200
elif data.bozo and \
data.bozo_exception.__class__.__name__.lower()=='timeout':
data.status = 408
else:
data.status = 500
activity_horizon = \
time.gmtime(time.time()-86400*config.activity_threshold(feed_uri))
# process based on the HTTP status code
if data.status == 200 and data.has_key("url"):
feed_info.feed['planet_http_location'] = data.url
if data.has_key("entries") and len(data.entries) == 0:
log.warning("No data %s", feed_uri)
feed_info.feed['planet_message'] = 'no data'
elif feed_uri == data.url:
log.info("Updating feed %s", feed_uri)
else:
log.info("Updating feed %s @ %s", feed_uri, data.url)
elif data.status == 301 and data.has_key("entries") and len(
data.entries) > 0:
log.warning("Feed has moved from <%s> to <%s>", feed_uri, data.url)
data.feed['planet_http_location'] = data.url
elif data.status == 304 and data.has_key("url"):
feed_info.feed['planet_http_location'] = data.url
if feed_uri == data.url:
log.info("Feed %s unchanged", feed_uri)
else:
log.info("Feed %s unchanged @ %s", feed_uri, data.url)
if not feed_info.feed.has_key('planet_message'):
if feed_info.feed.has_key('planet_updated'):
updated = feed_info.feed.planet_updated
if feedparser._parse_date_iso8601(updated) >= activity_horizon:
return
else:
if feed_info.feed.planet_message.startswith("no activity in"):
return
if not feed_info.feed.planet_message.startswith("duplicate") and \
not feed_info.feed.planet_message.startswith("no data"):
del feed_info.feed['planet_message']
elif data.status == 410:
log.info("Feed %s gone", feed_uri)
elif data.status == 408:
log.warning("Feed %s timed out", feed_uri)
elif data.status >= 400:
log.error("Error %d while updating feed %s", data.status, feed_uri)
else:
log.info("Updating feed %s", feed_uri)
# if read failed, retain cached information
if not data.version and feed_info.version:
data.feed = feed_info.feed
data.bozo = feed_info.feed.get('planet_bozo', 'true') == 'true'
data.version = feed_info.feed.get('planet_format')
data.feed['planet_http_status'] = str(data.status)
# capture etag and last-modified information
if data.has_key('headers'):
if data.has_key('etag') and data.etag:
data.feed['planet_http_etag'] = data.etag
elif data.headers.has_key('etag') and data.headers['etag']:
data.feed['planet_http_etag'] = data.headers['etag']
if data.headers.has_key('last-modified'):
data.feed['planet_http_last_modified'] = data.headers[
'last-modified']
elif data.has_key('modified') and data.modified:
data.feed['planet_http_last_modified'] = time.asctime(
data.modified)
if data.headers.has_key('-content-hash'):
data.feed['planet_content_hash'] = data.headers['-content-hash']
# capture feed and data from the planet configuration file
if data.version:
if not data.feed.has_key('links'): data.feed['links'] = list()
feedtype = 'application/atom+xml'
if data.version.startswith('rss'): feedtype = 'application/rss+xml'
if data.version in ['rss090', 'rss10']:
feedtype = 'application/rdf+xml'
for link in data.feed.links:
if link.rel == 'self':
link['type'] = feedtype
break
else:
data.feed.links.append(
feedparser.FeedParserDict({
'rel': 'self',
'type': feedtype,
'href': feed_uri
}))
for name, value in config.feed_options(feed_uri).items():
data.feed['planet_' + name] = value
# perform user configured scrub operations on the data
scrub.scrub(feed_uri, data)
from planet import idindex
global index
if index != None: index = idindex.open()
# select latest entry for each unique id
ids = {}
for entry in data.entries:
# generate an id, if none is present
if not entry.has_key('id') or not entry.id:
entry['id'] = reconstitute.id(None, entry)
if not entry['id']: continue
# determine updated date for purposes of selection
updated = ''
if entry.has_key('published'): updated = entry.published
if entry.has_key('updated'): updated = entry.updated
# if not seen or newer than last seen, select it
if updated >= ids.get(entry.id, ('', ))[0]:
ids[entry.id] = (updated, entry)
# write each entry to the cache
cache = config.cache_directory()
for updated, entry in ids.values():
# compute cache file name based on the id
cache_file = filename(cache, entry.id)
# get updated-date either from the entry or the cache (default to now)
mtime = None
#FEEDWORLD: published preferred over updated
if not entry.has_key(
'published_parsed') or not entry['published_parsed']:
entry['published_parsed'] = entry.get('updated_parsed', None)
if entry.has_key('published_parsed'):
try:
mtime = calendar.timegm(entry.published_parsed)
except:
pass
if not mtime:
try:
mtime = os.stat(cache_file).st_mtime
except:
if data.feed.has_key('published_parsed'):
try:
mtime = calendar.timegm(data.feed.published_parsed)
except:
pass
if not mtime: mtime = time.time()
entry['updated_parsed'] = time.gmtime(mtime)
# apply any filters
xdoc = reconstitute.reconstitute(data, entry)
output = xdoc.toxml().encode('utf-8')
xdoc.unlink()
for filter in config.filters(feed_uri):
output = shell.run(filter, output, mode="filter")
if not output: break
if not output:
if os.path.exists(cache_file): os.remove(cache_file)
continue
#FEEDWORLD: ugly hack to get around feeds that don't have timestamps
try:
elements = minidom.parseString(output).getElementsByTagName(
"feedworld_mtime")
if len(elements) > 0:
mtime = int(elements[0].firstChild.data)
#log.info("mtime=%d\n"%mtime)
except Exception, e:
log.debug("error: %s\n" % e)
# write out and timestamp the results
write(output, cache_file)
os.utime(cache_file, (mtime, mtime))
# optionally index
if index != None:
feedid = data.feed.get('id', data.feed.get('link', None))
if feedid:
if type(feedid) == unicode: feedid = feedid.encode('utf-8')
index[filename('', entry.id)] = feedid
def writeCache(feed_uri, feed_info, data):
log = planet.logger
sources = config.cache_sources_directory()
blacklist = config.cache_blacklist_directory()
# capture http status
if not data.has_key("status"):
if data.has_key("entries") and len(data.entries) > 0:
data.status = 200
elif data.bozo and \
data.bozo_exception.__class__.__name__.lower()=='timeout':
data.status = 408
else:
data.status = 500
activity_horizon = \
time.gmtime(time.time()-86400*config.activity_threshold(feed_uri))
# process based on the HTTP status code
if data.status == 200 and data.has_key("url"):
feed_info.feed['planet_http_location'] = data.url
if data.has_key("entries") and len(data.entries) == 0:
log.warning("No data %s", feed_uri)
feed_info.feed['planet_message'] = 'no data'
elif feed_uri == data.url:
log.info("Updating feed %s", feed_uri)
else:
log.info("Updating feed %s @ %s", feed_uri, data.url)
elif data.status == 301 and data.has_key("entries") and len(
data.entries) > 0:
log.warning("Feed has moved from <%s> to <%s>", feed_uri, data.url)
data.feed['planet_http_location'] = data.url
elif data.status == 304 and data.has_key("url"):
feed_info.feed['planet_http_location'] = data.url
if feed_uri == data.url:
log.info("Feed %s unchanged", feed_uri)
else:
log.info("Feed %s unchanged @ %s", feed_uri, data.url)
if not feed_info.feed.has_key('planet_message'):
if feed_info.feed.has_key('planet_updated'):
updated = feed_info.feed.planet_updated
if feedparser._parse_date_iso8601(updated) >= activity_horizon:
return
else:
if feed_info.feed.planet_message.startswith("no activity in"):
return
if not feed_info.feed.planet_message.startswith("duplicate") and \
not feed_info.feed.planet_message.startswith("no data"):
del feed_info.feed['planet_message']
elif data.status == 410:
log.info("Feed %s gone", feed_uri)
elif data.status == 408:
log.warning("Feed %s timed out", feed_uri)
elif data.status >= 400:
log.error("Error %d while updating feed %s", data.status, feed_uri)
else:
log.info("Updating feed %s", feed_uri)
# if read failed, retain cached information
if not data.get('version') and feed_info.get('version'):
data.feed = feed_info.feed
data.bozo = feed_info.feed.get('planet_bozo', 'true') == 'true'
data.version = feed_info.feed.get('planet_format')
data.feed['planet_http_status'] = str(data.status)
# capture etag and last-modified information
if data.has_key('headers'):
if data.has_key('etag') and data.etag:
data.feed['planet_http_etag'] = data.etag
elif data.headers.has_key('etag') and data.headers['etag']:
data.feed['planet_http_etag'] = data.headers['etag']
if data.headers.has_key('last-modified'):
data.feed['planet_http_last_modified'] = data.headers[
'last-modified']
elif data.has_key('modified') and data.modified:
data.feed['planet_http_last_modified'] = time.asctime(
data.modified)
if data.headers.has_key('-content-hash'):
data.feed['planet_content_hash'] = data.headers['-content-hash']
# capture feed and data from the planet configuration file
if data.get('version'):
if not data.feed.has_key('links'): data.feed['links'] = list()
feedtype = 'application/atom+xml'
if data.version.startswith('rss'): feedtype = 'application/rss+xml'
if data.version in ['rss090', 'rss10']:
feedtype = 'application/rdf+xml'
for link in data.feed.links:
if link.rel == 'self':
link['type'] = feedtype
break
else:
data.feed.links.append(
feedparser.FeedParserDict({
'rel': 'self',
'type': feedtype,
'href': feed_uri
}))
for name, value in config.feed_options(feed_uri).items():
if name == "gravatar" and re.match(re_email, value):
data.feed['planet_' + name] = md5(value.strip()).hexdigest()
else:
data.feed['planet_' + name] = value
# perform user configured scrub operations on the data
scrub.scrub(feed_uri, data)
from planet import idindex
global index
if index != None: index = idindex.open()
# select latest entry for each unique id
ids = {}
for entry in data.entries:
# generate an id, if none is present
if not entry.has_key('id') or not entry.id:
entry['id'] = reconstitute.id(None, entry)
elif hasattr(entry['id'], 'values'):
entry['id'] = entry['id'].values()[0]
if not entry['id']: continue
# determine updated date for purposes of selection
updated = ''
if entry.has_key('published'): updated = entry.published
if entry.has_key('updated'): updated = entry.updated
# if not seen or newer than last seen, select it
if updated >= ids.get(entry.id, ('', ))[0]:
ids[entry.id] = (updated, entry)
# write each entry to the cache
cache = config.cache_directory()
for updated, entry in ids.values():
# compute blacklist file name based on the id
blacklist_file = filename(blacklist, entry.id)
# check if blacklist file exists. If so, skip it.
if os.path.exists(blacklist_file):
continue
# compute cache file name based on the id
cache_file = filename(cache, entry.id)
# get updated-date either from the entry or the cache (default to now)
mtime = None
if not entry.has_key('updated_parsed') or not entry['updated_parsed']:
entry['updated_parsed'] = entry.get('published_parsed', None)
if entry.has_key('updated_parsed'):
try:
mtime = calendar.timegm(entry.updated_parsed)
except:
pass
if not mtime:
try:
mtime = os.stat(cache_file).st_mtime
except:
if data.feed.has_key('updated_parsed'):
try:
mtime = calendar.timegm(data.feed.updated_parsed)
except:
pass
if not mtime: mtime = time.time()
entry['updated_parsed'] = time.gmtime(mtime)
# apply any filters
xdoc = reconstitute.reconstitute(data, entry)
output = xdoc.toxml().encode('utf-8')
xdoc.unlink()
for filter in config.filters(feed_uri):
output = shell.run(filter, output, mode="filter")
if not output: break
if not output:
if os.path.exists(cache_file): os.remove(cache_file)
continue
# write out and timestamp the results
write(output, cache_file, mtime)
# optionally index
if index != None:
feedid = data.feed.get('id', data.feed.get('link', None))
if feedid:
if type(feedid) == unicode: feedid = feedid.encode('utf-8')
index[filename('', entry.id)] = feedid
if index: index.close()
# identify inactive feeds
if config.activity_threshold(feed_uri):
updated = [
entry.updated_parsed for entry in data.entries
if entry.has_key('updated_parsed')
]
updated.sort()
if updated:
data.feed['planet_updated'] = \
time.strftime("%Y-%m-%dT%H:%M:%SZ", updated[-1])
elif data.feed.has_key('planet_updated'):
updated = [
feedparser._parse_date_iso8601(data.feed.planet_updated)
]
if not updated or updated[-1] < activity_horizon:
msg = "no activity in %d days" % config.activity_threshold(
feed_uri)
log.info(msg)
data.feed['planet_message'] = msg
# report channel level errors
if data.status == 226:
if data.feed.has_key('planet_message'): del data.feed['planet_message']
if feed_info.feed.has_key('planet_updated'):
data.feed['planet_updated'] = feed_info.feed['planet_updated']
elif data.status == 403:
data.feed['planet_message'] = "403: forbidden"
elif data.status == 404:
data.feed['planet_message'] = "404: not found"
elif data.status == 408:
data.feed['planet_message'] = "408: request timeout"
elif data.status == 410:
data.feed['planet_message'] = "410: gone"
elif data.status == 500:
data.feed['planet_message'] = "internal server error"
elif data.status >= 400:
data.feed['planet_message'] = "http status %s" % data.status
# write the feed info to the cache
if not os.path.exists(sources): os.makedirs(sources)
xdoc = minidom.parseString('''<feed xmlns:planet="%s"
xmlns="http://www.w3.org/2005/Atom"/>\n''' % planet.xmlns)
reconstitute.source(xdoc.documentElement, data.feed, data.bozo,
data.version)
write(xdoc.toxml().encode('utf-8'), filename(sources, feed_uri))
xdoc.unlink()
def writeCache(feed_uri, feed_info, data):
log = planet.logger
sources = config.cache_sources_directory()
blacklist = config.cache_blacklist_directory()
# capture http status
if not data.has_key("status"):
if data.has_key("entries") and len(data.entries)>0:
data.status = 200
elif data.bozo and \
data.bozo_exception.__class__.__name__.lower()=='timeout':
data.status = 408
else:
data.status = 500
activity_horizon = \
time.gmtime(time.time()-86400*config.activity_threshold(feed_uri))
# process based on the HTTP status code
if data.status == 200 and data.has_key("url"):
feed_info.feed['planet_http_location'] = data.url
if data.has_key("entries") and len(data.entries) == 0:
log.warning("No data %s", feed_uri)
feed_info.feed['planet_message'] = 'no data'
elif feed_uri == data.url:
log.info("Updating feed %s", feed_uri)
else:
log.info("Updating feed %s @ %s", feed_uri, data.url)
elif data.status == 301 and data.has_key("entries") and len(data.entries)>0:
log.warning("Feed has moved from <%s> to <%s>", feed_uri, data.url)
data.feed['planet_http_location'] = data.url
elif data.status == 304 and data.has_key("url"):
feed_info.feed['planet_http_location'] = data.url
if feed_uri == data.url:
log.info("Feed %s unchanged", feed_uri)
else:
log.info("Feed %s unchanged @ %s", feed_uri, data.url)
if not feed_info.feed.has_key('planet_message'):
if feed_info.feed.has_key('planet_updated'):
updated = feed_info.feed.planet_updated
if feedparser._parse_date_iso8601(updated) >= activity_horizon:
return
else:
if feed_info.feed.planet_message.startswith("no activity in"):
return
if not feed_info.feed.planet_message.startswith("duplicate") and \
not feed_info.feed.planet_message.startswith("no data"):
del feed_info.feed['planet_message']
elif data.status == 410:
log.info("Feed %s gone", feed_uri)
elif data.status == 408:
log.warning("Feed %s timed out", feed_uri)
elif data.status >= 400:
log.error("Error %d while updating feed %s", data.status, feed_uri)
else:
log.info("Updating feed %s", feed_uri)
# if read failed, retain cached information
if not data.get('version') and feed_info.get('version'):
data.feed = feed_info.feed
data.bozo = feed_info.feed.get('planet_bozo','true') == 'true'
data.version = feed_info.feed.get('planet_format')
data.feed['planet_http_status'] = str(data.status)
# capture etag and last-modified information
if data.has_key('headers'):
if data.has_key('etag') and data.etag:
data.feed['planet_http_etag'] = data.etag
elif data.headers.has_key('etag') and data.headers['etag']:
data.feed['planet_http_etag'] = data.headers['etag']
if data.headers.has_key('last-modified'):
data.feed['planet_http_last_modified']=data.headers['last-modified']
elif data.has_key('modified') and data.modified:
data.feed['planet_http_last_modified'] = time.asctime(data.modified)
if data.headers.has_key('-content-hash'):
data.feed['planet_content_hash'] = data.headers['-content-hash']
# capture feed and data from the planet configuration file
if data.get('version'):
if not data.feed.has_key('links'): data.feed['links'] = list()
feedtype = 'application/atom+xml'
if data.version.startswith('rss'): feedtype = 'application/rss+xml'
if data.version in ['rss090','rss10']: feedtype = 'application/rdf+xml'
for link in data.feed.links:
if link.rel == 'self':
link['type'] = feedtype
break
else:
data.feed.links.append(feedparser.FeedParserDict(
{'rel':'self', 'type':feedtype, 'href':feed_uri}))
for name, value in config.feed_options(feed_uri).items():
data.feed['planet_'+name] = value
# perform user configured scrub operations on the data
scrub.scrub(feed_uri, data)
from planet import idindex
global index
if index != None: index = idindex.open()
# select latest entry for each unique id
ids = {}
for entry in data.entries:
# generate an id, if none is present
if not entry.has_key('id') or not entry.id:
entry['id'] = reconstitute.id(None, entry)
elif hasattr(entry['id'], 'values'):
entry['id'] = entry['id'].values()[0]
if not entry['id']: continue
# determine updated date for purposes of selection
updated = ''
if entry.has_key('published'): updated=entry.published
if entry.has_key('updated'): updated=entry.updated
# if not seen or newer than last seen, select it
if updated >= ids.get(entry.id,('',))[0]:
ids[entry.id] = (updated, entry)
# write each entry to the cache
cache = config.cache_directory()
for updated, entry in ids.values():
# compute blacklist file name based on the id
blacklist_file = filename(blacklist, entry.id)
# check if blacklist file exists. If so, skip it.
if os.path.exists(blacklist_file):
continue
# compute cache file name based on the id
cache_file = filename(cache, entry.id)
# get updated-date either from the entry or the cache (default to now)
mtime = None
if not entry.has_key('updated_parsed') or not entry['updated_parsed']:
entry['updated_parsed'] = entry.get('published_parsed',None)
if entry.has_key('updated_parsed'):
try:
mtime = calendar.timegm(entry.updated_parsed)
except:
pass
if not mtime:
try:
mtime = os.stat(cache_file).st_mtime
except:
if data.feed.has_key('updated_parsed'):
try:
mtime = calendar.timegm(data.feed.updated_parsed)
except:
pass
if not mtime: mtime = time.time()
entry['updated_parsed'] = time.gmtime(mtime)
# apply any filters
xdoc = reconstitute.reconstitute(data, entry)
output = xdoc.toxml().encode('utf-8')
xdoc.unlink()
for filter in config.filters(feed_uri):
output = shell.run(filter, output, mode="filter")
if not output: break
if not output:
if os.path.exists(cache_file): os.remove(cache_file)
continue
# write out and timestamp the results
write(output, cache_file, mtime)
# optionally index
if index != None:
feedid = data.feed.get('id', data.feed.get('link',None))
if feedid:
if type(feedid) == unicode: feedid = feedid.encode('utf-8')
index[filename('', entry.id)] = feedid
if index: index.close()
# identify inactive feeds
if config.activity_threshold(feed_uri):
updated = [entry.updated_parsed for entry in data.entries
if entry.has_key('updated_parsed')]
updated.sort()
if updated:
data.feed['planet_updated'] = \
time.strftime("%Y-%m-%dT%H:%M:%SZ", updated[-1])
elif data.feed.has_key('planet_updated'):
updated = [feedparser._parse_date_iso8601(data.feed.planet_updated)]
if not updated or updated[-1] < activity_horizon:
msg = "no activity in %d days" % config.activity_threshold(feed_uri)
log.info(msg)
data.feed['planet_message'] = msg
# report channel level errors
if data.status == 226:
if data.feed.has_key('planet_message'): del data.feed['planet_message']
if feed_info.feed.has_key('planet_updated'):
data.feed['planet_updated'] = feed_info.feed['planet_updated']
elif data.status == 403:
data.feed['planet_message'] = "403: forbidden"
elif data.status == 404:
data.feed['planet_message'] = "404: not found"
elif data.status == 408:
data.feed['planet_message'] = "408: request timeout"
elif data.status == 410:
data.feed['planet_message'] = "410: gone"
elif data.status == 500:
data.feed['planet_message'] = "internal server error"
elif data.status >= 400:
data.feed['planet_message'] = "http status %s" % data.status
# write the feed info to the cache
if not os.path.exists(sources): os.makedirs(sources)
xdoc=minidom.parseString('''<feed xmlns:planet="%s"
xmlns="http://www.w3.org/2005/Atom"/>\n''' % planet.xmlns)
reconstitute.source(xdoc.documentElement,data.feed,data.bozo,data.version)
write(xdoc.toxml().encode('utf-8'), filename(sources, feed_uri))
xdoc.unlink()