def post(self):
# allow single feed to be updated
key = self.request.get("key")
if key is not None and len(key) > 0:
feed = FeedStream.get(db.Key(key))
if feed is None:
self.response.out.write("no feed to update")
return
self.update_feed(feed)
self.response.out.write("feed updated")
return
# no key do the recursive polling
is_enabled = FeedPollerConfig.get_instance().is_enabled
if is_enabled is None or is_enabled == False:
logging.info("feed poller not enabled shutting down")
self.response.out.write("feed poller not enabled shutting down")
return
# Get the stalest feed
feed = FeedStream.all().filter('deleted = ', False).order('last_polled').get()
# Check how stale the stalest feed is, if it's been updated in the
# last 10 minutes, we should take a break
max_age = datetime.utcnow() - timedelta(minutes=10)
if feed.last_polled >= max_age:
if not memcache.set("feed_poller_running", False):
logging.error("memcache set failed")
logging.info("putting feed poller to sleep")
self.response.out.write("putting feed poller to sleep")
return
# go get the feed
try:
self.update_feed(feed)
except:
logging.warn("Update Failed for feed with stream_id %s and url: %s" % (feed.stream_id, feed.url))
logging.error(sys.exc_info()[0])
try:
feed.last_polled = datetime.utcnow()
feed.has_error = True
feed.put()
except:
logging.warn("failed to save feed error")
# Queue the next feed
task = taskqueue.Task(url='/feedpoller/tasks/poll', params={}).add(queue_name="feed-poller")
self.response.out.write("feed updated")
def get(self, stream_id):
mode = self.request.GET['hub.mode']
topic = self.request.GET['hub.topic']
challenge = self.request.GET['hub.challenge']
verify_token = self.request.GET['hub.verify_token']
feedstream = FeedStream.get_by_key_name("z%s" % stream_id)
if feedstream is None:
logging.warn("feedstream not found in pshb subscription callback: %s" % topic)
self.error(404)
return
if mode == 'unsubscribe':
logging.info("pshb unsubscribe callback for %s" % topic)
feedstream.pshb_is_subscribed = False
feedstream.put()
self.response.headers['Content-Type'] = 'text/plain'
self.response.out.write(challenge)
return
if mode != 'subscribe':
logging.warn("pshb mode unknown %s" % mode)
self.error(400)
return
if feedstream.pshb_verify_token != verify_token:
logging.warn("verify token's don't match. topic: %s verify_token: %s" % (topic, verify_token))
self.error(400)
return
logging.info("pshb topic subscription callback for %s" % topic)
feedstream.pshb_is_subscribed = True
feedstream.put()
self.response.headers['Content-Type'] = 'text/plain'
self.response.out.write(challenge)
def post(self):
stream = FeedStream.get(db.Key(self.request.POST.get("key")))
if stream is None:
logging.warn("feedstream not found for subscription request")
self.response.out.write("feedstream not found for subscription request")
self.error(404)
return
feed = feedparser.parse(stream.url)
if hasattr(feed, 'feed') and hasattr(feed.feed, 'links'):
hub_url = find_feed_url('hub', feed.feed.links)
if hub_url is None:
logging.info("no hub found for: %s" % stream.url)
self.response.out.write('no hub found')
return
else:
logging.info("sending pshb subscription request for: %s" % stream.url)
stream.pshb_hub_url = hub_url
stream.put()
self.subscribe_to_topic(stream, hub_url)
self.response.out.write('sent subscription request')
return
logging.warn('could not parse feed unable to initiate subscription')
self.response.out.write('could not parse feed unable to initiate subscription')
self.error(400)
def post(self, stream_id):
"""Handles Content Distribution notifications."""
logging.debug(self.request.headers)
logging.debug(self.request.body)
feed = feedparser.parse(self.request.body)
if feed.bozo:
logging.error('Bozo feed data. %s: %r',
feed.bozo_exception.__class__.__name__,
feed.bozo_exception)
if (hasattr(feed.bozo_exception, 'getLineNumber') and
hasattr(feed.bozo_exception, 'getMessage')):
line = feed.bozo_exception.getLineNumber()
logging.error('Line %d: %s', line, feed.bozo_exception.getMessage())
segment = self.request.body.split('\n')[line-1]
logging.info('Body segment with error: %r', segment.decode('utf-8'))
return self.response.set_status(500)
feedstream = FeedStream.get_by_key_name("z%s" % stream_id)
if feedstream is None:
logging.warn("Discarding update from unknown feed '%s'", stream_id)
self.error(404)
return
logging.info("Processing update for feed '%s'", feedstream.url)
logging.info('Found %d entries', len(feed.entries))
to_put = [] # batch datastore updates
for entry in feed.entries:
item = FeedItem.process_entry(entry, feedstream)
if item is not None:
to_put.append(item)
if len(to_put) > 0:
db.put(to_put)
# update feed last_polled or http_last_modified so feed poller doesn't have to check this feed for a while
feedstream.last_polled = datetime.utcnow()
feedstream.put()
#self.update_mavenn_activity(feedstream.stream_id, to_put)
# Response headers (body can be empty)
# X-Hub-On-Behalf-Of
self.response.set_status(200)
self.response.out.write("ok");
def post(self, key):
stream = FeedStream.get_by_key_name("z%s" % key)
db.delete(stream.items)
stream.deleted = True
stream.put()
self.redirect('/admin/')
def get(self, key):
stream = FeedStream.get_by_key_name("z%s" % key)
self.generate('admin/delete.html', { "stream": stream })
def get(self, key):
stream = FeedStream.get_by_key_name("z%s" % key)
items = stream.items.order('-updated').fetch(10)
self.generate('admin/view.html', { "items": items, "stream": stream })
def get(self):
streams = FeedStream.all().order('-created').fetch(200)
self.generate('admin/list.html', { "streams": streams })
def list(self):
"""List the streams created for this bot"""
streams = FeedStream.all().filter('deleted =', False).fetch(100)
self.generate('streams.html', {"streams": streams, "title": "Feed Streams", "bot_path": "feeds"})
def get_stream(self, stream_id):
return FeedStream.get_by_key_name("z%s" % stream_id)
