Exemplo n.º 1
0
def deferred_ratings():
    """ This is the deferred ratings table calculation process """
    # Disable the in-context cache to save memory
    # (it doesn't give any speed advantage for this processing)
    Context.disable_cache()
    t0 = time.time()

    try:

        _create_ratings()

    except DeadlineExceededError as ex:
        # Hit deadline: save the stuff we already have and
        # defer a new task to continue where we left off
        logging.error(u"Deadline exceeded in ratings, failing permamently")
        # Normal return prevents this task from being run again
        raise deferred.PermanentTaskFailure()

    except Exception as ex:
        logging.error(
            u"Exception in ratings, failing permanently: {0}".format(ex))
        # Avoid having the task retried
        raise deferred.PermanentTaskFailure()

    t1 = time.time()

    logging.info(u"Ratings calculation finished in {0:.2f} seconds".format(t1 -
                                                                           t0))
    StatsModel.log_cache_stats()
    StatsModel.clear_cache(
    )  # Do not maintain the cache in memory between runs
Exemplo n.º 2
0
def deferred_update():
    """ Update all users in the datastore with lowercase nick and full name """
    logging.info("Deferred user update starting")
    CHUNK_SIZE = 200
    count = 0
    offset = 0
    Context.disable_cache()
    try:
        q = UserModel.query()
        while True:
            ulist = []
            chunk = 0
            for um in q.fetch(CHUNK_SIZE, offset = offset):
                chunk += 1
                if um.nick_lc is None:
                    try:
                        um.nick_lc = um.nickname.lower()
                        um.name_lc = um.prefs.get("full_name", "").lower() if um.prefs else ""
                        ulist.append(um)
                    except Exception as e:
                        logging.info("Exception in deferred_update() when setting nick_lc: {0}".format(e))
            if ulist:
                try:
                    ndb.put_multi(ulist)
                    count += len(ulist)
                except Exception as e:
                    logging.info("Exception in deferred_update() when updating ndb: {0}".format(e))
            if chunk < CHUNK_SIZE:
                break
            offset += CHUNK_SIZE
    except Exception as e:
        logging.info("Exception in deferred_update(): {0}, already updated {1} records".format(e, count))
        # Do not retry the task
        raise deferred.PermanentTaskFailure()
    logging.info("Completed updating {0} user records".format(count))
Exemplo n.º 3
0
def deferred_ratings():
    """ This is the deferred ratings table calculation process """
    # Disable the in-context cache to save memory
    # (it doesn't give any speed advantage for this processing)
    Context.disable_cache()
    t0 = time.time()

    try:

        _create_ratings()

    except DeadlineExceededError as ex:
        # Hit deadline: save the stuff we already have and
        # defer a new task to continue where we left off
        logging.error(u"Deadline exceeded in ratings, failing permamently")
        # Normal return prevents this task from being run again
        raise deferred.PermanentTaskFailure()

    except Exception as ex:
        logging.error(u"Exception in ratings, failing permanently: {0}".format(ex))
        # Avoid having the task retried
        raise deferred.PermanentTaskFailure()

    t1 = time.time()

    logging.info(u"Ratings calculation finished in {0:.2f} seconds".format(t1 - t0))
    StatsModel.log_cache_stats()
    StatsModel.clear_cache() # Do not maintain the cache in memory between runs
Exemplo n.º 4
0
def deferred_update():
    """ Update all users in the datastore with lowercase nick and full name """
    logging.info("Deferred user update starting")
    CHUNK_SIZE = 200
    count = 0
    offset = 0
    Context.disable_cache()
    try:
        q = UserModel.query()
        while True:
            ulist = []
            chunk = 0
            for um in q.fetch(CHUNK_SIZE, offset = offset):
                chunk += 1
                if um.nick_lc is None:
                    try:
                        um.nick_lc = um.nickname.lower()
                        um.name_lc = um.prefs.get("full_name", "").lower() if um.prefs else ""
                        ulist.append(um)
                    except Exception as e:
                        logging.info("Exception in deferred_update() when setting nick_lc: {0}".format(e))
            if ulist:
                try:
                    ndb.put_multi(ulist)
                    count += len(ulist)
                except Exception as e:
                    logging.info("Exception in deferred_update() when updating ndb: {0}".format(e))
            if chunk < CHUNK_SIZE:
                break
            offset += CHUNK_SIZE
    except Exception as e:
        logging.info("Exception in deferred_update(): {0}, already updated {1} records".format(e, count))
        # Do not retry the task
        raise deferred.PermanentTaskFailure()
    logging.info("Completed updating {0} user records".format(count))
Exemplo n.º 5
0
def deferred_ratings(timestamp):
    """ This is the deferred ratings table calculation process """
    # Disable the in-context cache to save memory
    # (it doesn't give any speed advantage for this processing)
    Context.disable_cache()

    t0 = time.time()
    _create_ratings(timestamp)
    t1 = time.time()

    logging.info(u"Ratings calculation finished in {0:.2f} seconds".format(t1 - t0))
Exemplo n.º 6
0
def deferred_stats(from_time, to_time):
    """ This is the deferred stats collection process """
    # Disable the in-context cache to save memory
    # (it doesn't give any speed advantage for this processing)
    Context.disable_cache()

    t0 = time.time()
    _run_stats(from_time, to_time)
    t1 = time.time()

    logging.info(u"Stats calculation finished in {0:.2f} seconds".format(t1 - t0))
Exemplo n.º 7
0
def deferred_stats(from_time, to_time):
    """ This is the deferred stats collection process """
    # Disable the in-context cache to save memory
    # (it doesn't give any speed advantage for this processing)
    Context.disable_cache()

    t0 = time.time()
    _run_stats(from_time, to_time)
    t1 = time.time()

    logging.info(u"Stats calculation finished in {0:.2f} seconds".format(t1 -
                                                                         t0))
Exemplo n.º 8
0
def deferred_ratings(timestamp):
    """ This is the deferred ratings table calculation process """
    # Disable the in-context cache to save memory
    # (it doesn't give any speed advantage for this processing)
    Context.disable_cache()

    t0 = time.time()
    _create_ratings(timestamp)
    t1 = time.time()

    logging.info(u"Ratings calculation finished in {0:.2f} seconds".format(t1 -
                                                                           t0))