def update_feed(self, feed):
"""Fetch the feed and process new items"""
d = self.parse_feed(feed)
if d is None:
logging.warn("Parsing failed")
feed.last_polled = datetime.utcnow()
feed.put()
return
to_put = []
for entry in d['entries']:
item = FeedItem.process_entry(entry, feed)
if item is not None:
item_exists = FeedItem.get_by_key_name(item._key_name)
if item_exists is None:
to_put.append(item)
# TODO: what about updates?
if len(to_put) > 0:
db.put(to_put)
# self.update_mavenn_activity(feed.stream_id, to_put)
# update feedstream properties
if hasattr(d, 'status'):
feed.http_status = str(d.status)
if hasattr(d, 'modified'):
feed.http_last_modified = datetime(*d.modified[:6])
if hasattr(d, 'etag'):
feed.http_etag = d.etag
feed.last_polled = datetime.utcnow()
feed.put()
return
def post(self, stream_id):
"""Handles Content Distribution notifications."""
logging.debug(self.request.headers)
logging.debug(self.request.body)
feed = feedparser.parse(self.request.body)
if feed.bozo:
logging.error('Bozo feed data. %s: %r',
feed.bozo_exception.__class__.__name__,
feed.bozo_exception)
if (hasattr(feed.bozo_exception, 'getLineNumber') and
hasattr(feed.bozo_exception, 'getMessage')):
line = feed.bozo_exception.getLineNumber()
logging.error('Line %d: %s', line, feed.bozo_exception.getMessage())
segment = self.request.body.split('\n')[line-1]
logging.info('Body segment with error: %r', segment.decode('utf-8'))
return self.response.set_status(500)
feedstream = FeedStream.get_by_key_name("z%s" % stream_id)
if feedstream is None:
logging.warn("Discarding update from unknown feed '%s'", stream_id)
self.error(404)
return
logging.info("Processing update for feed '%s'", feedstream.url)
logging.info('Found %d entries', len(feed.entries))
to_put = [] # batch datastore updates
for entry in feed.entries:
item = FeedItem.process_entry(entry, feedstream)
if item is not None:
to_put.append(item)
if len(to_put) > 0:
db.put(to_put)
# update feed last_polled or http_last_modified so feed poller doesn't have to check this feed for a while
feedstream.last_polled = datetime.utcnow()
feedstream.put()
#self.update_mavenn_activity(feedstream.stream_id, to_put)
# Response headers (body can be empty)
# X-Hub-On-Behalf-Of
self.response.set_status(200)
self.response.out.write("ok");
