Exemplo n.º 1
0
def deferred_ratings():
    """ This is the deferred ratings table calculation process """
    # Disable the in-context cache to save memory
    # (it doesn't give any speed advantage for this processing)
    Context.disable_cache()
    t0 = time.time()

    try:

        _create_ratings()

    except DeadlineExceededError as ex:
        # Hit deadline: save the stuff we already have and
        # defer a new task to continue where we left off
        logging.error(u"Deadline exceeded in ratings, failing permamently")
        # Normal return prevents this task from being run again
        raise deferred.PermanentTaskFailure()

    except Exception as ex:
        logging.error(u"Exception in ratings, failing permanently: {0}".format(ex))
        # Avoid having the task retried
        raise deferred.PermanentTaskFailure()

    t1 = time.time()

    logging.info(u"Ratings calculation finished in {0:.2f} seconds".format(t1 - t0))
    StatsModel.log_cache_stats()
    StatsModel.clear_cache() # Do not maintain the cache in memory between runs
Exemplo n.º 2
0
def deferred_ratings():
    """ This is the deferred ratings table calculation process """
    # Disable the in-context cache to save memory
    # (it doesn't give any speed advantage for this processing)
    Context.disable_cache()
    t0 = time.time()

    try:

        _create_ratings()

    except DeadlineExceededError as ex:
        # Hit deadline: save the stuff we already have and
        # defer a new task to continue where we left off
        logging.error(u"Deadline exceeded in ratings, failing permamently")
        # Normal return prevents this task from being run again
        raise deferred.PermanentTaskFailure()

    except Exception as ex:
        logging.error(
            u"Exception in ratings, failing permanently: {0}".format(ex))
        # Avoid having the task retried
        raise deferred.PermanentTaskFailure()

    t1 = time.time()

    logging.info(u"Ratings calculation finished in {0:.2f} seconds".format(t1 -
                                                                           t0))
    StatsModel.log_cache_stats()
    StatsModel.clear_cache(
    )  # Do not maintain the cache in memory between runs
Exemplo n.º 3
0
 def statistics(self):
     """ Return a set of key statistics on the user """
     reply = dict()
     sm = StatsModel.newest_for_user(self.id())
     reply["result"] = Error.LEGAL
     reply["nickname"] = self.nickname()
     reply["fullname"] = self.full_name()
     sm.populate_dict(reply)
     return reply
Exemplo n.º 4
0
def deferred_ratings():
    """ This is the deferred ratings table calculation process """

    with Client.get_context() as context:

        t0 = time.time()
        try:
            _create_ratings()
        except Exception as ex:
            logging.error("Exception in deferred_ratings: {0!r}".format(ex))
            return
        t1 = time.time()

        StatsModel.log_cache_stats()
        # Do not maintain the cache in memory between runs
        StatsModel.clear_cache()

        logging.info(
            "Ratings calculation finished in {0:.2f} seconds".format(t1 - t0))
Exemplo n.º 5
0
def _write_stats(timestamp, urecs):
    """ Writes the freshly calculated statistics records to the database """
    # Delete all previous stats with the same timestamp, if any
    StatsModel.delete_ts(timestamp = timestamp)
    um_list = []
    for sm in urecs.values():
        # Set the reference timestamp for the entire stats series
        sm.timestamp = timestamp
        # Fetch user information to update Elo statistics
        if sm.user:
            # Not robot
            um = UserModel.fetch(sm.user.id())
            if um:
                um.elo = sm.elo
                um.human_elo = sm.human_elo
                um_list.append(um)
    # Update the statistics records
    StatsModel.put_multi(urecs.values())
    # Update the user records
    UserModel.put_multi(um_list)
Exemplo n.º 6
0
def _write_stats(timestamp, urecs):
    """ Writes the freshly calculated statistics records to the database """
    # Delete all previous stats with the same timestamp, if any
    StatsModel.delete_ts(timestamp=timestamp)
    um_list = []
    for sm in urecs.values():
        # Set the reference timestamp for the entire stats series
        sm.timestamp = timestamp
        # Fetch user information to update Elo statistics
        if sm.user:
            # Not robot
            um = UserModel.fetch(sm.user.id())
            if um:
                um.elo = sm.elo
                um.human_elo = sm.human_elo
                um_list.append(um)
    # Update the statistics records
    StatsModel.put_multi(urecs.values())
    # Update the user records
    UserModel.put_multi(um_list)
Exemplo n.º 7
0
 def statistics(self):
     """ Return a set of key statistics on the user """
     reply = dict()
     sm = StatsModel.newest_for_user(self.id())
     reply["result"] = Error.LEGAL
     reply["nickname"] = self.nickname()
     reply["fullname"] = self.full_name()
     sm.populate_dict(reply)
     # Add statistics from the user entity
     reply["highest_score"] = self._highest_score
     reply["highest_score_game"] = self._highest_score_game
     reply["best_word"] = self._best_word
     reply["best_word_score"] = self._best_word_score
     reply["best_word_game"] = self._best_word_game
     return reply
Exemplo n.º 8
0
 def statistics(self):
     """ Return a set of key statistics on the user """
     reply = dict()
     sm = StatsModel.newest_for_user(self.id())
     reply["result"] = Error.LEGAL
     reply["nickname"] = self.nickname()
     reply["fullname"] = self.full_name()
     sm.populate_dict(reply)
     # Add statistics from the user entity
     reply["highest_score"] = self._highest_score
     reply["highest_score_game"] = self._highest_score_game
     reply["best_word"] = self._best_word
     reply["best_word_score"] = self._best_word_score
     reply["best_word_game"] = self._best_word_game
     return reply
Exemplo n.º 9
0
def _create_ratings(timestamp):
    """ Create the Top 100 ratings tables """

    logging.info(u"Starting _create_ratings")

    _key = StatsModel.dict_key

    def _augment_table(t, t_yesterday, t_week_ago, t_month_ago):
        """ Go through a table of top scoring users and augment it with data from previous time points """

        for sm in t:
            # Augment the rating with info about progress
            key = _key(sm)

            def _augment(prop):
                sm[prop + "_yesterday"] = t_yesterday[key][prop] if key in t_yesterday else 0
                sm[prop + "_week_ago"] = t_week_ago[key][prop] if key in t_week_ago else 0
                sm[prop + "_month_ago"] = t_month_ago[key][prop] if key in t_month_ago else 0

            _augment("rank")
            _augment("games")
            _augment("elo")
            _augment("wins")
            _augment("losses")
            _augment("score")
            _augment("score_against")

    # All players including robot games

    top100_all = [ sm for sm in StatsModel.list_elo(timestamp, 100) ]
    top100_all_yesterday = { _key(sm) : sm for sm in StatsModel.list_elo(timestamp - timedelta(days = 1), 100) }
    top100_all_week_ago = { _key(sm) : sm for sm in StatsModel.list_elo(timestamp - timedelta(days = 7), 100) }
    top100_all_month_ago = { _key(sm) : sm for sm in StatsModel.list_elo(monthdelta(timestamp, -1), 100) }

    # Augment the table for all games
    _augment_table(top100_all, top100_all_yesterday, top100_all_week_ago, top100_all_month_ago)

    # Human only games

    top100_human = [ sm for sm in StatsModel.list_human_elo(timestamp, 100) ]
    top100_human_yesterday = { _key(sm) : sm for sm in StatsModel.list_human_elo(timestamp - timedelta(days = 1), 100) }
    top100_human_week_ago = { _key(sm) : sm for sm in StatsModel.list_human_elo(timestamp - timedelta(days = 7), 100) }
    top100_human_month_ago = { _key(sm) : sm for sm in StatsModel.list_human_elo(monthdelta(timestamp, -1), 100) }

    # Augment the table for human only games
    _augment_table(top100_human, top100_human_yesterday, top100_human_week_ago, top100_human_month_ago)

    logging.info(u"Writing top 100 tables to the database")

    # Write the Top 100 tables to the database
    for rank in range(0, 100):

        # All players including robots
        rm = RatingModel.get_or_create("all", rank + 1)
        if rank < len(top100_all):
            rm.assign(top100_all[rank])
        else:
            # Sentinel empty records
            rm.user = None
            rm.robot_level = -1
            rm.games = -1
        rm.put()

        # Humans only
        rm = RatingModel.get_or_create("human", rank + 1)
        if rank < len(top100_human):
            rm.assign(top100_human[rank])
        else:
            # Sentinel empty records
            rm.user = None
            rm.robot_level = -1
            rm.games = -1
        rm.put()

    logging.info(u"Finishing _create_ratings")
Exemplo n.º 10
0
 def _init_stat(user_id, robot_level):
     """ Returns the newest StatsModel instance available for the given user """
     return StatsModel.newest_before(from_time, user_id, robot_level)
Exemplo n.º 11
0
def _create_ratings():
    """ Create the Top 100 ratings tables """

    logging.info(u"Starting _create_ratings")

    _key = StatsModel.dict_key

    timestamp = datetime.utcnow()
    yesterday = timestamp - timedelta(days=1)
    week_ago = timestamp - timedelta(days=7)
    month_ago = monthdelta(timestamp, -1)

    def _augment_table(t, t_yesterday, t_week_ago, t_month_ago):
        """ Go through a table of top scoring users and augment it with data from previous time points """

        for sm in t:
            # Augment the rating with info about progress
            key = _key(sm)

            def _augment(prop):
                sm[prop + "_yesterday"] = t_yesterday[key][
                    prop] if key in t_yesterday else 0
                sm[prop + "_week_ago"] = t_week_ago[key][
                    prop] if key in t_week_ago else 0
                sm[prop + "_month_ago"] = t_month_ago[key][
                    prop] if key in t_month_ago else 0

            _augment("rank")
            _augment("games")
            _augment("elo")
            _augment("wins")
            _augment("losses")
            _augment("score")
            _augment("score_against")

    # All players including robot games

    #top100_all = StatsModel.list_elo(None, 100)
    top100_all = [sm for sm in StatsModel.list_elo(timestamp, 100)]
    top100_all_yesterday = {
        _key(sm): sm
        for sm in StatsModel.list_elo(yesterday, 100)
    }
    top100_all_week_ago = {
        _key(sm): sm
        for sm in StatsModel.list_elo(week_ago, 100)
    }
    top100_all_month_ago = {
        _key(sm): sm
        for sm in StatsModel.list_elo(month_ago, 100)
    }

    # Augment the table for all games
    _augment_table(top100_all, top100_all_yesterday, top100_all_week_ago,
                   top100_all_month_ago)

    # Human only games

    #top100_human = StatsModel.list_human_elo(None, 100)
    top100_human = [sm for sm in StatsModel.list_human_elo(timestamp, 100)]
    top100_human_yesterday = {
        _key(sm): sm
        for sm in StatsModel.list_human_elo(yesterday, 100)
    }
    top100_human_week_ago = {
        _key(sm): sm
        for sm in StatsModel.list_human_elo(week_ago, 100)
    }
    top100_human_month_ago = {
        _key(sm): sm
        for sm in StatsModel.list_human_elo(month_ago, 100)
    }

    # Augment the table for human only games
    _augment_table(top100_human, top100_human_yesterday, top100_human_week_ago,
                   top100_human_month_ago)

    logging.info(u"Writing top 100 tables to the database")

    # Write the Top 100 tables to the database
    rlist = []

    for rank in range(0, 100):

        # All players including robots
        rm = RatingModel.get_or_create("all", rank + 1)
        if rank < len(top100_all):
            rm.assign(top100_all[rank])
        else:
            # Sentinel empty records
            rm.user = None
            rm.robot_level = -1
            rm.games = -1
        rlist.append(rm)

        # Humans only
        rm = RatingModel.get_or_create("human", rank + 1)
        if rank < len(top100_human):
            rm.assign(top100_human[rank])
        else:
            # Sentinel empty records
            rm.user = None
            rm.robot_level = -1
            rm.games = -1
        rlist.append(rm)

    # Put the entire top 100 table in one RPC call
    RatingModel.put_multi(rlist)

    logging.info(u"Finishing _create_ratings")
Exemplo n.º 12
0
 def _init_stat(user_id, robot_level):
     """ Returns the newest StatsModel instance available for the given user """
     return StatsModel.newest_before(from_time, user_id, robot_level)
Exemplo n.º 13
0
def _run_stats(from_time, to_time):
    """ Runs a process to update user statistics and Elo ratings """
    logging.info("Generating stats from {0} to {1}".format(from_time, to_time))

    if from_time is None or to_time is None:
        # Time range must be specified
        return False

    if from_time >= to_time:
        # Null time range
        return False

    # Clear previous cache contents, if any
    StatsModel.clear_cache()

    # Iterate over all finished games within the time span in temporal order
    # pylint: disable=singleton-comparison
    q = (GameModel.query(
        ndb.AND(GameModel.ts_last_move > from_time,
                GameModel.ts_last_move <= to_time)).order(
                    GameModel.ts_last_move).filter(GameModel.over == True))

    # The accumulated user statistics
    users = dict()

    def _init_stat(user_id, robot_level):
        """ Returns the newest StatsModel instance available for the given user """
        return StatsModel.newest_before(from_time, user_id, robot_level)

    cnt = 0
    ts_last_processed = None

    try:
        # Use i as a progress counter
        i = 0
        for gm in iter_q(q, chunk_size=250):
            i += 1
            lm = Alphabet.format_timestamp(gm.ts_last_move or gm.timestamp)
            p0 = None if gm.player0 is None else gm.player0.id()
            p1 = None if gm.player1 is None else gm.player1.id()
            robot_game = (p0 is None) or (p1 is None)
            if robot_game:
                rl = gm.robot_level
            else:
                rl = 0
            s0 = gm.score0
            s1 = gm.score1

            if (s0 == 0) and (s1 == 0):
                # When a game ends by resigning immediately,
                # make sure that the weaker player
                # doesn't get Elo points for a draw; in fact,
                # ignore such a game altogether in the statistics
                continue

            if p0 is None:
                k0 = "robot-" + str(rl)
            else:
                k0 = p0
            if p1 is None:
                k1 = "robot-" + str(rl)
            else:
                k1 = p1

            if k0 in users:
                urec0 = users[k0]
            else:
                users[k0] = urec0 = _init_stat(p0, rl if p0 is None else 0)
            if k1 in users:
                urec1 = users[k1]
            else:
                users[k1] = urec1 = _init_stat(p1, rl if p1 is None else 0)
            # Number of games played
            urec0.games += 1
            urec1.games += 1
            if not robot_game:
                urec0.human_games += 1
                urec1.human_games += 1
            # Total scores
            urec0.score += s0
            urec1.score += s1
            urec0.score_against += s1
            urec1.score_against += s0
            if not robot_game:
                urec0.human_score += s0
                urec1.human_score += s1
                urec0.human_score_against += s1
                urec1.human_score_against += s0
            # Wins and losses
            if s0 > s1:
                urec0.wins += 1
                urec1.losses += 1
            elif s1 > s0:
                urec1.wins += 1
                urec0.losses += 1
            if not robot_game:
                if s0 > s1:
                    urec0.human_wins += 1
                    urec1.human_losses += 1
                elif s1 > s0:
                    urec1.human_wins += 1
                    urec0.human_losses += 1
            # Find out whether players are established or beginners
            est0 = urec0.games > ESTABLISHED_MARK
            est1 = urec1.games > ESTABLISHED_MARK
            # Save the Elo point state used in the calculation
            gm.elo0, gm.elo1 = urec0.elo, urec1.elo
            # Compute the Elo points of both players
            adj = _compute_elo((urec0.elo, urec1.elo), s0, s1, est0, est1)
            # When an established player is playing a beginning (provisional) player,
            # leave the Elo score of the established player unchanged
            # Adjust player 0
            if est0 and not est1:
                adj = (0, adj[1])
            gm.elo0_adj = adj[0]
            urec0.elo += adj[0]
            # Adjust player 1
            if est1 and not est0:
                adj = (adj[0], 0)
            gm.elo1_adj = adj[1]
            urec1.elo += adj[1]
            # If not a robot game, compute the human-only Elo
            if not robot_game:
                gm.human_elo0, gm.human_elo1 = urec0.human_elo, urec1.human_elo
                adj = _compute_elo((urec0.human_elo, urec1.human_elo), s0, s1,
                                   est0, est1)
                # Adjust player 0
                if est0 and not est1:
                    adj = (0, adj[1])
                gm.human_elo0_adj = adj[0]
                urec0.human_elo += adj[0]
                # Adjust player 1
                if est1 and not est0:
                    adj = (adj[0], 0)
                gm.human_elo1_adj = adj[1]
                urec1.human_elo += adj[1]
            # Save the game object with the new Elo adjustment statistics
            gm.put()
            # Save the last processed timestamp
            ts_last_processed = lm
            cnt += 1
            # Report on our progress
            if i % 500 == 0:
                logging.info("Processed {0} games".format(i))

    except Exception as ex:
        logging.error(
            "Exception in _run_stats() after {0} games and {1} users: {2!r}".
            format(cnt, len(users), ex))
        return False

    # Completed without incident
    logging.info(
        "Normal completion of stats for {1} games and {0} users".format(
            len(users), cnt))
    _write_stats(to_time, users)
    return True