def post(self): # allow single feed to be updated key = self.request.get("key") if key is not None and len(key) > 0: feed = FeedStream.get(db.Key(key)) if feed is None: self.response.out.write("no feed to update") return self.update_feed(feed) self.response.out.write("feed updated") return # no key do the recursive polling is_enabled = FeedPollerConfig.get_instance().is_enabled if is_enabled is None or is_enabled == False: logging.info("feed poller not enabled shutting down") self.response.out.write("feed poller not enabled shutting down") return # Get the stalest feed feed = FeedStream.all().filter('deleted = ', False).order('last_polled').get() # Check how stale the stalest feed is, if it's been updated in the # last 10 minutes, we should take a break max_age = datetime.utcnow() - timedelta(minutes=10) if feed.last_polled >= max_age: if not memcache.set("feed_poller_running", False): logging.error("memcache set failed") logging.info("putting feed poller to sleep") self.response.out.write("putting feed poller to sleep") return # go get the feed try: self.update_feed(feed) except: logging.warn("Update Failed for feed with stream_id %s and url: %s" % (feed.stream_id, feed.url)) logging.error(sys.exc_info()[0]) try: feed.last_polled = datetime.utcnow() feed.has_error = True feed.put() except: logging.warn("failed to save feed error") # Queue the next feed task = taskqueue.Task(url='/feedpoller/tasks/poll', params={}).add(queue_name="feed-poller") self.response.out.write("feed updated")
def get(self, stream_id): mode = self.request.GET['hub.mode'] topic = self.request.GET['hub.topic'] challenge = self.request.GET['hub.challenge'] verify_token = self.request.GET['hub.verify_token'] feedstream = FeedStream.get_by_key_name("z%s" % stream_id) if feedstream is None: logging.warn("feedstream not found in pshb subscription callback: %s" % topic) self.error(404) return if mode == 'unsubscribe': logging.info("pshb unsubscribe callback for %s" % topic) feedstream.pshb_is_subscribed = False feedstream.put() self.response.headers['Content-Type'] = 'text/plain' self.response.out.write(challenge) return if mode != 'subscribe': logging.warn("pshb mode unknown %s" % mode) self.error(400) return if feedstream.pshb_verify_token != verify_token: logging.warn("verify token's don't match. topic: %s verify_token: %s" % (topic, verify_token)) self.error(400) return logging.info("pshb topic subscription callback for %s" % topic) feedstream.pshb_is_subscribed = True feedstream.put() self.response.headers['Content-Type'] = 'text/plain' self.response.out.write(challenge)
def post(self): stream = FeedStream.get(db.Key(self.request.POST.get("key"))) if stream is None: logging.warn("feedstream not found for subscription request") self.response.out.write("feedstream not found for subscription request") self.error(404) return feed = feedparser.parse(stream.url) if hasattr(feed, 'feed') and hasattr(feed.feed, 'links'): hub_url = find_feed_url('hub', feed.feed.links) if hub_url is None: logging.info("no hub found for: %s" % stream.url) self.response.out.write('no hub found') return else: logging.info("sending pshb subscription request for: %s" % stream.url) stream.pshb_hub_url = hub_url stream.put() self.subscribe_to_topic(stream, hub_url) self.response.out.write('sent subscription request') return logging.warn('could not parse feed unable to initiate subscription') self.response.out.write('could not parse feed unable to initiate subscription') self.error(400)
def post(self, stream_id): """Handles Content Distribution notifications.""" logging.debug(self.request.headers) logging.debug(self.request.body) feed = feedparser.parse(self.request.body) if feed.bozo: logging.error('Bozo feed data. %s: %r', feed.bozo_exception.__class__.__name__, feed.bozo_exception) if (hasattr(feed.bozo_exception, 'getLineNumber') and hasattr(feed.bozo_exception, 'getMessage')): line = feed.bozo_exception.getLineNumber() logging.error('Line %d: %s', line, feed.bozo_exception.getMessage()) segment = self.request.body.split('\n')[line-1] logging.info('Body segment with error: %r', segment.decode('utf-8')) return self.response.set_status(500) feedstream = FeedStream.get_by_key_name("z%s" % stream_id) if feedstream is None: logging.warn("Discarding update from unknown feed '%s'", stream_id) self.error(404) return logging.info("Processing update for feed '%s'", feedstream.url) logging.info('Found %d entries', len(feed.entries)) to_put = [] # batch datastore updates for entry in feed.entries: item = FeedItem.process_entry(entry, feedstream) if item is not None: to_put.append(item) if len(to_put) > 0: db.put(to_put) # update feed last_polled or http_last_modified so feed poller doesn't have to check this feed for a while feedstream.last_polled = datetime.utcnow() feedstream.put() #self.update_mavenn_activity(feedstream.stream_id, to_put) # Response headers (body can be empty) # X-Hub-On-Behalf-Of self.response.set_status(200) self.response.out.write("ok");
def post(self, key): stream = FeedStream.get_by_key_name("z%s" % key) db.delete(stream.items) stream.deleted = True stream.put() self.redirect('/admin/')
def get(self, key): stream = FeedStream.get_by_key_name("z%s" % key) self.generate('admin/delete.html', { "stream": stream })
def get(self, key): stream = FeedStream.get_by_key_name("z%s" % key) items = stream.items.order('-updated').fetch(10) self.generate('admin/view.html', { "items": items, "stream": stream })
def get(self): streams = FeedStream.all().order('-created').fetch(200) self.generate('admin/list.html', { "streams": streams })
def list(self): """List the streams created for this bot""" streams = FeedStream.all().filter('deleted =', False).fetch(100) self.generate('streams.html', {"streams": streams, "title": "Feed Streams", "bot_path": "feeds"})
def get_stream(self, stream_id): return FeedStream.get_by_key_name("z%s" % stream_id)