示例#1
0
def post_all_feeds():
    """Post all new items for feeds for a specific interval"""
    if request.headers.get("X-Appengine-Cron") != "true":
        raise ndb.Return(jsonify_error(message="Not a cron call"))

    logger.info("Starting a post job")
    futures = []
    for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems():
        feeds = feed_class.query(feed_class.is_dirty == True)
        logger.info("Got some feeds_count: %s feeds_type: %s", feeds.count(), feed_type)
        success = 0
        more = True
        cursor = None
        while more:
            feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor)
            keys = ",".join([x.key.urlsafe() for x in feeds_to_fetch])
            if not keys:
                continue
            futures.append(
                Queue().add_async(Task(url=url_for("tq_feed_post-canonical"), method="POST", params={"keys": keys}))
            )
            success += len(feeds_to_fetch)
        logger.info("queued post for %d feeds feed_type:%s", success, feed_type)

    for future in futures:
        yield future

    logger.info("Finished Post Job")
    yield write_epoch_to_stat(Stat, "post_job")
    raise ndb.Return(jsonify(status="ok"))
示例#2
0
def update_all_feeds(interval_id):
    """Update all feeds for a specific interval"""
    if request.headers.get("X-Appengine-Cron") != "true":
        raise ndb.Return(jsonify_error(message="Not a cron call"))

    for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems():
        feeds = Feed.for_interval(interval_id)
        success = 0
        more = True
        cursor = None
        futures = []
        while more:
            feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor)
            feeds_to_fetch = filter(
                lambda x: getattr(x, "external_polling_bucket", DEFAULT_POLLING_BUCKET) == DEFAULT_POLLING_BUCKET,
                feeds_to_fetch,
            )
            keys = ",".join([x.key.urlsafe() for x in feeds_to_fetch])
            if not keys:
                continue

            futures.append(
                Queue("poll").add_async(
                    Task(url=url_for("tq_feed_poll-canonical"), method="POST", params={"keys": keys})
                )
            )
            success += 1

    for future in futures:
        yield future

    logger.info("queued poll for %d feeds at interval_id=%s", success, interval_id)

    raise ndb.Return(jsonify(status="ok"))
示例#3
0
def post_all_feeds():
    """Post all new items for feeds for a specific interval"""
    if request.headers.get('X-Appengine-Cron') != 'true':
        raise ndb.Return(jsonify_error(message='Not a cron call'))

    logger.info('Starting a post job')
    futures = []
    for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems():
        feeds = feed_class.query(feed_class.is_dirty == True)
        logger.info("Got some feeds_count: %s feeds_type: %s", feeds.count(), feed_type)
        success = 0
        more = True
        cursor = None
        while more:
            feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor)
            keys = ','.join([x.key.urlsafe() for x in feeds_to_fetch])
            if not keys:
                continue
            futures.append(Queue().add_async(Task(url=url_for('tq_feed_post-canonical'), method='POST', params={'keys': keys})))
            success += len(feeds_to_fetch)
        logger.info('queued post for %d feeds feed_type:%s', success, feed_type)

    for future in futures:
        yield future

    logger.info('Finished Post Job')
    yield write_epoch_to_stat(Stat, 'post_job')
    raise ndb.Return(jsonify(status='ok'))
示例#4
0
def update_all_feeds(interval_id):
    """Update all feeds for a specific interval"""
    if request.headers.get('X-Appengine-Cron') != 'true':
        raise ndb.Return(jsonify_error(message='Not a cron call'))

    for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems():
        feeds = Feed.for_interval(interval_id)
        success = 0
        more = True
        cursor = None
        futures = []
        while more:
            feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor)
            feeds_to_fetch = filter(lambda x: getattr(x, 'external_polling_bucket', DEFAULT_POLLING_BUCKET) == DEFAULT_POLLING_BUCKET, feeds_to_fetch)
            keys = ','.join([x.key.urlsafe() for x in feeds_to_fetch])
            if not keys:
                continue

            futures.append(Queue('poll').add_async(Task(url=url_for('tq_feed_poll-canonical'), method='POST', params={'keys': keys})))
            success += 1

    for future in futures:
        yield future

    logger.info('queued poll for %d feeds at interval_id=%s', success, interval_id)

    raise ndb.Return(jsonify(status='ok'))
示例#5
0
def feeds():
    """List all examples"""
    users_feeds = []
    for feed_type in FEED_TYPE_TO_CLASS.values():
        users_feeds += [
            feed.to_json() for feed in feed_type.for_user(g.user)
            if feed.visible
        ]
    return jsonify(status='ok', data=users_feeds)
示例#6
0
def feeds():
    """List all examples"""
    users_feeds = []
    for feed_type in FEED_TYPE_TO_CLASS.values():
        users_feeds += [feed.to_json() for feed in feed_type.for_user(g.user) if feed.visible]
    return jsonify(status='ok', data=users_feeds)