def all_feeds(): """Post all new items for feeds for a specific interval""" def feed_to_dict(feed): return { "feed_key": feed.key.urlsafe(), "feed_url": feed.feed_url, "etag": feed.etag, "last_hash": feed.last_fetched_content_hash, "update_interval": UPDATE_INTERVAL_TO_MINUTES.get(feed.update_interval), } bucket = int(request.args.get("bucket_id", 1)) feed_clss = [Feed, RssFeed] feeds_response = [] for feed_cls in feed_clss: qit = feed_cls.query(feed_cls.external_polling_bucket == bucket) more = True cursor = None while more: feeds_to_fetch, cursor, more = yield qit.fetch_page_async(1000, start_cursor=cursor) feeds_response.extend((feed_to_dict(feed) for feed in feeds_to_fetch)) poller_run_id = uuid.uuid4().hex logger.info("Poller run %s dispatched with %d feeds", poller_run_id, len(feeds_response)) response = {"poller_run_id": poller_run_id, "feeds": feeds_response} yield write_epoch_to_stat(Stat, "external_poll_get_all_feeds") raise ndb.Return(jsonify(status="ok", data=response))
def post_all_feeds(): """Post all new items for feeds for a specific interval""" if request.headers.get('X-Appengine-Cron') != 'true': raise ndb.Return(jsonify_error(message='Not a cron call')) logger.info('Starting a post job') futures = [] for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems(): feeds = feed_class.query(feed_class.is_dirty == True) logger.info("Got some feeds_count: %s feeds_type: %s", feeds.count(), feed_type) success = 0 more = True cursor = None while more: feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor) keys = ','.join([x.key.urlsafe() for x in feeds_to_fetch]) if not keys: continue futures.append(Queue().add_async(Task(url=url_for('tq_feed_post-canonical'), method='POST', params={'keys': keys}))) success += len(feeds_to_fetch) logger.info('queued post for %d feeds feed_type:%s', success, feed_type) for future in futures: yield future logger.info('Finished Post Job') yield write_epoch_to_stat(Stat, 'post_job') raise ndb.Return(jsonify(status='ok'))
def post_all_feeds(): """Post all new items for feeds for a specific interval""" if request.headers.get("X-Appengine-Cron") != "true": raise ndb.Return(jsonify_error(message="Not a cron call")) logger.info("Starting a post job") futures = [] for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems(): feeds = feed_class.query(feed_class.is_dirty == True) logger.info("Got some feeds_count: %s feeds_type: %s", feeds.count(), feed_type) success = 0 more = True cursor = None while more: feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor) keys = ",".join([x.key.urlsafe() for x in feeds_to_fetch]) if not keys: continue futures.append( Queue().add_async(Task(url=url_for("tq_feed_post-canonical"), method="POST", params={"keys": keys})) ) success += len(feeds_to_fetch) logger.info("queued post for %d feeds feed_type:%s", success, feed_type) for future in futures: yield future logger.info("Finished Post Job") yield write_epoch_to_stat(Stat, "post_job") raise ndb.Return(jsonify(status="ok"))
def tq_feed_poll(): """Poll some feeds feed""" if not request.headers.get("X-AppEngine-QueueName"): raise ndb.Return(jsonify_error(message="Not a Task call")) keys = request.form.get("keys") if not keys: logger.info("Task Queue poll no keys") raise ndb.Return(jsonify_error(code=500)) success = 0 errors = 0 entries_created = 0 ndb_keys = [ndb.Key(urlsafe=key) for key in keys.split(",")] feeds = yield ndb.get_multi_async(ndb_keys) feeds = filter(lambda x: not getattr(x, "use_external_poller", False), feeds) logger.info("Got %d feed(s) for polling", len(feeds)) futures = [] for i, feed in enumerate(feeds): if not feed: errors += 1 logger.info("Couldn't find feed for key: %s", ndb_keys[i]) continue futures.append((i, feed.process_feed(None, None))) for i, future in futures: parsed_feed = None try: parsed_feed, num_new_entries = yield future entries_created += num_new_entries success += 1 except: errors += 1 feed = feeds[i] logger.exception("Failed to update feed:%s, i=%s" % (feed.feed_url, i)) yield write_epoch_to_stat(Stat, "poll_job") logger.info("Polled feeds entries_created: %s success: %s errors: %s", entries_created, success, errors) raise ndb.Return(jsonify(status="ok"))
def tq_feed_poll(): """Poll some feeds feed""" if not request.headers.get('X-AppEngine-QueueName'): raise ndb.Return(jsonify_error(message='Not a Task call')) keys = request.form.get('keys') if not keys: logger.info('Task Queue poll no keys') raise ndb.Return(jsonify_error(code=500)) success = 0 errors = 0 entries_created = 0 ndb_keys = [ndb.Key(urlsafe=key) for key in keys.split(',')] feeds = yield ndb.get_multi_async(ndb_keys) feeds = filter(lambda x: not getattr(x, 'use_external_poller', False), feeds) logger.info('Got %d feed(s) for polling', len(feeds)) futures = [] for i, feed in enumerate(feeds): if not feed: errors += 1 logger.info("Couldn't find feed for key: %s", ndb_keys[i]) continue futures.append((i, feed.process_feed(None, None))) for i, future in futures: parsed_feed = None try: parsed_feed, num_new_entries = yield future entries_created += num_new_entries success += 1 except: errors += 1 feed = feeds[i] logger.exception('Failed to update feed:%s, i=%s' % (feed.feed_url, i)) yield write_epoch_to_stat(Stat, 'poll_job') logger.info('Polled feeds entries_created: %s success: %s errors: %s', entries_created, success, errors) raise ndb.Return(jsonify(status='ok'))
def feed_push_update_app(feed_key): feed = ndb.Key(urlsafe=feed_key).get() if not feed: raise ndb.Return(jsonify_error("Unknown feed")) noop = request.args.get("noop") if noop: logger.info("Noop feed publish %s because off testing", feed_key) raise ndb.Return(jsonify(status="ok")) post_data = { "feed_key": feed_key, "feed_data": request.stream.read(), "etag": request.args.get("etag"), "last_hash": request.args.get("last_hash"), } yield inbound_feed_process(**post_data) # yield Queue('inbound-posts').add_async(Task(url=url_for('tq_inbound_feed'), method='POST', params=post_data)) yield write_epoch_to_stat(Stat, "external_poll_post_feed") raise ndb.Return(jsonify(status="ok"))
def feed_push_update_app(feed_key): feed = ndb.Key(urlsafe=feed_key).get() if not feed: raise ndb.Return(jsonify_error('Unknown feed')) noop = request.args.get('noop') if noop: logger.info('Noop feed publish %s because off testing', feed_key) raise ndb.Return(jsonify(status='ok')) post_data = { 'feed_key': feed_key, 'feed_data': request.stream.read(), 'etag': request.args.get('etag'), 'last_hash': request.args.get('last_hash'), } yield inbound_feed_process(**post_data) # yield Queue('inbound-posts').add_async(Task(url=url_for('tq_inbound_feed'), method='POST', params=post_data)) yield write_epoch_to_stat(Stat, 'external_poll_post_feed') raise ndb.Return(jsonify(status='ok'))
def all_feeds(): """Post all new items for feeds for a specific interval""" def feed_to_dict(feed): return { 'feed_key': feed.key.urlsafe(), 'feed_url': feed.feed_url, 'etag': feed.etag, 'last_hash': feed.last_fetched_content_hash, 'update_interval': UPDATE_INTERVAL_TO_MINUTES.get(feed.update_interval) } bucket = int(request.args.get('bucket_id', 1)) feed_clss = [Feed, RssFeed] feeds_response = [] for feed_cls in feed_clss: qit = feed_cls.query(feed_cls.external_polling_bucket == bucket) more = True cursor = None while more: feeds_to_fetch, cursor, more = yield qit.fetch_page_async(1000, start_cursor=cursor) feeds_response.extend((feed_to_dict(feed) for feed in feeds_to_fetch)) poller_run_id = uuid.uuid4().hex logger.info('Poller run %s dispatched with %d feeds', poller_run_id, len(feeds_response)) response = { 'poller_run_id': poller_run_id, 'feeds': feeds_response, } yield write_epoch_to_stat(Stat, 'external_poll_get_all_feeds') raise ndb.Return(jsonify(status='ok', data=response))