def all_feeds():
"""Post all new items for feeds for a specific interval"""
def feed_to_dict(feed):
return {
"feed_key": feed.key.urlsafe(),
"feed_url": feed.feed_url,
"etag": feed.etag,
"last_hash": feed.last_fetched_content_hash,
"update_interval": UPDATE_INTERVAL_TO_MINUTES.get(feed.update_interval),
}
bucket = int(request.args.get("bucket_id", 1))
feed_clss = [Feed, RssFeed]
feeds_response = []
for feed_cls in feed_clss:
qit = feed_cls.query(feed_cls.external_polling_bucket == bucket)
more = True
cursor = None
while more:
feeds_to_fetch, cursor, more = yield qit.fetch_page_async(1000, start_cursor=cursor)
feeds_response.extend((feed_to_dict(feed) for feed in feeds_to_fetch))
poller_run_id = uuid.uuid4().hex
logger.info("Poller run %s dispatched with %d feeds", poller_run_id, len(feeds_response))
response = {"poller_run_id": poller_run_id, "feeds": feeds_response}
yield write_epoch_to_stat(Stat, "external_poll_get_all_feeds")
raise ndb.Return(jsonify(status="ok", data=response))
def post_all_feeds():
"""Post all new items for feeds for a specific interval"""
if request.headers.get('X-Appengine-Cron') != 'true':
raise ndb.Return(jsonify_error(message='Not a cron call'))
logger.info('Starting a post job')
futures = []
for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems():
feeds = feed_class.query(feed_class.is_dirty == True)
logger.info("Got some feeds_count: %s feeds_type: %s", feeds.count(), feed_type)
success = 0
more = True
cursor = None
while more:
feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor)
keys = ','.join([x.key.urlsafe() for x in feeds_to_fetch])
if not keys:
continue
futures.append(Queue().add_async(Task(url=url_for('tq_feed_post-canonical'), method='POST', params={'keys': keys})))
success += len(feeds_to_fetch)
logger.info('queued post for %d feeds feed_type:%s', success, feed_type)
for future in futures:
yield future
logger.info('Finished Post Job')
yield write_epoch_to_stat(Stat, 'post_job')
raise ndb.Return(jsonify(status='ok'))
def post_all_feeds():
"""Post all new items for feeds for a specific interval"""
if request.headers.get("X-Appengine-Cron") != "true":
raise ndb.Return(jsonify_error(message="Not a cron call"))
logger.info("Starting a post job")
futures = []
for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems():
feeds = feed_class.query(feed_class.is_dirty == True)
logger.info("Got some feeds_count: %s feeds_type: %s", feeds.count(), feed_type)
success = 0
more = True
cursor = None
while more:
feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor)
keys = ",".join([x.key.urlsafe() for x in feeds_to_fetch])
if not keys:
continue
futures.append(
Queue().add_async(Task(url=url_for("tq_feed_post-canonical"), method="POST", params={"keys": keys}))
)
success += len(feeds_to_fetch)
logger.info("queued post for %d feeds feed_type:%s", success, feed_type)
for future in futures:
yield future
logger.info("Finished Post Job")
yield write_epoch_to_stat(Stat, "post_job")
raise ndb.Return(jsonify(status="ok"))
def tq_feed_poll():
"""Poll some feeds feed"""
if not request.headers.get("X-AppEngine-QueueName"):
raise ndb.Return(jsonify_error(message="Not a Task call"))
keys = request.form.get("keys")
if not keys:
logger.info("Task Queue poll no keys")
raise ndb.Return(jsonify_error(code=500))
success = 0
errors = 0
entries_created = 0
ndb_keys = [ndb.Key(urlsafe=key) for key in keys.split(",")]
feeds = yield ndb.get_multi_async(ndb_keys)
feeds = filter(lambda x: not getattr(x, "use_external_poller", False), feeds)
logger.info("Got %d feed(s) for polling", len(feeds))
futures = []
for i, feed in enumerate(feeds):
if not feed:
errors += 1
logger.info("Couldn't find feed for key: %s", ndb_keys[i])
continue
futures.append((i, feed.process_feed(None, None)))
for i, future in futures:
parsed_feed = None
try:
parsed_feed, num_new_entries = yield future
entries_created += num_new_entries
success += 1
except:
errors += 1
feed = feeds[i]
logger.exception("Failed to update feed:%s, i=%s" % (feed.feed_url, i))
yield write_epoch_to_stat(Stat, "poll_job")
logger.info("Polled feeds entries_created: %s success: %s errors: %s", entries_created, success, errors)
raise ndb.Return(jsonify(status="ok"))
def tq_feed_poll():
"""Poll some feeds feed"""
if not request.headers.get('X-AppEngine-QueueName'):
raise ndb.Return(jsonify_error(message='Not a Task call'))
keys = request.form.get('keys')
if not keys:
logger.info('Task Queue poll no keys')
raise ndb.Return(jsonify_error(code=500))
success = 0
errors = 0
entries_created = 0
ndb_keys = [ndb.Key(urlsafe=key) for key in keys.split(',')]
feeds = yield ndb.get_multi_async(ndb_keys)
feeds = filter(lambda x: not getattr(x, 'use_external_poller', False), feeds)
logger.info('Got %d feed(s) for polling', len(feeds))
futures = []
for i, feed in enumerate(feeds):
if not feed:
errors += 1
logger.info("Couldn't find feed for key: %s", ndb_keys[i])
continue
futures.append((i, feed.process_feed(None, None)))
for i, future in futures:
parsed_feed = None
try:
parsed_feed, num_new_entries = yield future
entries_created += num_new_entries
success += 1
except:
errors += 1
feed = feeds[i]
logger.exception('Failed to update feed:%s, i=%s' % (feed.feed_url, i))
yield write_epoch_to_stat(Stat, 'poll_job')
logger.info('Polled feeds entries_created: %s success: %s errors: %s', entries_created, success, errors)
raise ndb.Return(jsonify(status='ok'))
def feed_push_update_app(feed_key):
feed = ndb.Key(urlsafe=feed_key).get()
if not feed:
raise ndb.Return(jsonify_error("Unknown feed"))
noop = request.args.get("noop")
if noop:
logger.info("Noop feed publish %s because off testing", feed_key)
raise ndb.Return(jsonify(status="ok"))
post_data = {
"feed_key": feed_key,
"feed_data": request.stream.read(),
"etag": request.args.get("etag"),
"last_hash": request.args.get("last_hash"),
}
yield inbound_feed_process(**post_data)
# yield Queue('inbound-posts').add_async(Task(url=url_for('tq_inbound_feed'), method='POST', params=post_data))
yield write_epoch_to_stat(Stat, "external_poll_post_feed")
raise ndb.Return(jsonify(status="ok"))
def feed_push_update_app(feed_key):
feed = ndb.Key(urlsafe=feed_key).get()
if not feed:
raise ndb.Return(jsonify_error('Unknown feed'))
noop = request.args.get('noop')
if noop:
logger.info('Noop feed publish %s because off testing', feed_key)
raise ndb.Return(jsonify(status='ok'))
post_data = {
'feed_key': feed_key,
'feed_data': request.stream.read(),
'etag': request.args.get('etag'),
'last_hash': request.args.get('last_hash'),
}
yield inbound_feed_process(**post_data)
# yield Queue('inbound-posts').add_async(Task(url=url_for('tq_inbound_feed'), method='POST', params=post_data))
yield write_epoch_to_stat(Stat, 'external_poll_post_feed')
raise ndb.Return(jsonify(status='ok'))
def all_feeds():
"""Post all new items for feeds for a specific interval"""
def feed_to_dict(feed):
return {
'feed_key': feed.key.urlsafe(),
'feed_url': feed.feed_url,
'etag': feed.etag,
'last_hash': feed.last_fetched_content_hash,
'update_interval': UPDATE_INTERVAL_TO_MINUTES.get(feed.update_interval)
}
bucket = int(request.args.get('bucket_id', 1))
feed_clss = [Feed, RssFeed]
feeds_response = []
for feed_cls in feed_clss:
qit = feed_cls.query(feed_cls.external_polling_bucket == bucket)
more = True
cursor = None
while more:
feeds_to_fetch, cursor, more = yield qit.fetch_page_async(1000, start_cursor=cursor)
feeds_response.extend((feed_to_dict(feed) for feed in feeds_to_fetch))
poller_run_id = uuid.uuid4().hex
logger.info('Poller run %s dispatched with %d feeds', poller_run_id, len(feeds_response))
response = {
'poller_run_id': poller_run_id,
'feeds': feeds_response,
}
yield write_epoch_to_stat(Stat, 'external_poll_get_all_feeds')
raise ndb.Return(jsonify(status='ok', data=response))
