def post_all_feeds(): """Post all new items for feeds for a specific interval""" if request.headers.get("X-Appengine-Cron") != "true": raise ndb.Return(jsonify_error(message="Not a cron call")) logger.info("Starting a post job") futures = [] for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems(): feeds = feed_class.query(feed_class.is_dirty == True) logger.info("Got some feeds_count: %s feeds_type: %s", feeds.count(), feed_type) success = 0 more = True cursor = None while more: feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor) keys = ",".join([x.key.urlsafe() for x in feeds_to_fetch]) if not keys: continue futures.append( Queue().add_async(Task(url=url_for("tq_feed_post-canonical"), method="POST", params={"keys": keys})) ) success += len(feeds_to_fetch) logger.info("queued post for %d feeds feed_type:%s", success, feed_type) for future in futures: yield future logger.info("Finished Post Job") yield write_epoch_to_stat(Stat, "post_job") raise ndb.Return(jsonify(status="ok"))
def update_all_feeds(interval_id): """Update all feeds for a specific interval""" if request.headers.get("X-Appengine-Cron") != "true": raise ndb.Return(jsonify_error(message="Not a cron call")) for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems(): feeds = Feed.for_interval(interval_id) success = 0 more = True cursor = None futures = [] while more: feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor) feeds_to_fetch = filter( lambda x: getattr(x, "external_polling_bucket", DEFAULT_POLLING_BUCKET) == DEFAULT_POLLING_BUCKET, feeds_to_fetch, ) keys = ",".join([x.key.urlsafe() for x in feeds_to_fetch]) if not keys: continue futures.append( Queue("poll").add_async( Task(url=url_for("tq_feed_poll-canonical"), method="POST", params={"keys": keys}) ) ) success += 1 for future in futures: yield future logger.info("queued poll for %d feeds at interval_id=%s", success, interval_id) raise ndb.Return(jsonify(status="ok"))
def post_all_feeds(): """Post all new items for feeds for a specific interval""" if request.headers.get('X-Appengine-Cron') != 'true': raise ndb.Return(jsonify_error(message='Not a cron call')) logger.info('Starting a post job') futures = [] for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems(): feeds = feed_class.query(feed_class.is_dirty == True) logger.info("Got some feeds_count: %s feeds_type: %s", feeds.count(), feed_type) success = 0 more = True cursor = None while more: feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor) keys = ','.join([x.key.urlsafe() for x in feeds_to_fetch]) if not keys: continue futures.append(Queue().add_async(Task(url=url_for('tq_feed_post-canonical'), method='POST', params={'keys': keys}))) success += len(feeds_to_fetch) logger.info('queued post for %d feeds feed_type:%s', success, feed_type) for future in futures: yield future logger.info('Finished Post Job') yield write_epoch_to_stat(Stat, 'post_job') raise ndb.Return(jsonify(status='ok'))
def update_all_feeds(interval_id): """Update all feeds for a specific interval""" if request.headers.get('X-Appengine-Cron') != 'true': raise ndb.Return(jsonify_error(message='Not a cron call')) for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems(): feeds = Feed.for_interval(interval_id) success = 0 more = True cursor = None futures = [] while more: feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor) feeds_to_fetch = filter(lambda x: getattr(x, 'external_polling_bucket', DEFAULT_POLLING_BUCKET) == DEFAULT_POLLING_BUCKET, feeds_to_fetch) keys = ','.join([x.key.urlsafe() for x in feeds_to_fetch]) if not keys: continue futures.append(Queue('poll').add_async(Task(url=url_for('tq_feed_poll-canonical'), method='POST', params={'keys': keys}))) success += 1 for future in futures: yield future logger.info('queued poll for %d feeds at interval_id=%s', success, interval_id) raise ndb.Return(jsonify(status='ok'))
def feeds(): """List all examples""" users_feeds = [] for feed_type in FEED_TYPE_TO_CLASS.values(): users_feeds += [ feed.to_json() for feed in feed_type.for_user(g.user) if feed.visible ] return jsonify(status='ok', data=users_feeds)
def feeds(): """List all examples""" users_feeds = [] for feed_type in FEED_TYPE_TO_CLASS.values(): users_feeds += [feed.to_json() for feed in feed_type.for_user(g.user) if feed.visible] return jsonify(status='ok', data=users_feeds)