def deferred_ratings(): """ This is the deferred ratings table calculation process """ # Disable the in-context cache to save memory # (it doesn't give any speed advantage for this processing) Context.disable_cache() t0 = time.time() try: _create_ratings() except DeadlineExceededError as ex: # Hit deadline: save the stuff we already have and # defer a new task to continue where we left off logging.error(u"Deadline exceeded in ratings, failing permamently") # Normal return prevents this task from being run again raise deferred.PermanentTaskFailure() except Exception as ex: logging.error(u"Exception in ratings, failing permanently: {0}".format(ex)) # Avoid having the task retried raise deferred.PermanentTaskFailure() t1 = time.time() logging.info(u"Ratings calculation finished in {0:.2f} seconds".format(t1 - t0)) StatsModel.log_cache_stats() StatsModel.clear_cache() # Do not maintain the cache in memory between runs
def deferred_ratings(): """ This is the deferred ratings table calculation process """ # Disable the in-context cache to save memory # (it doesn't give any speed advantage for this processing) Context.disable_cache() t0 = time.time() try: _create_ratings() except DeadlineExceededError as ex: # Hit deadline: save the stuff we already have and # defer a new task to continue where we left off logging.error(u"Deadline exceeded in ratings, failing permamently") # Normal return prevents this task from being run again raise deferred.PermanentTaskFailure() except Exception as ex: logging.error( u"Exception in ratings, failing permanently: {0}".format(ex)) # Avoid having the task retried raise deferred.PermanentTaskFailure() t1 = time.time() logging.info(u"Ratings calculation finished in {0:.2f} seconds".format(t1 - t0)) StatsModel.log_cache_stats() StatsModel.clear_cache( ) # Do not maintain the cache in memory between runs
def statistics(self): """ Return a set of key statistics on the user """ reply = dict() sm = StatsModel.newest_for_user(self.id()) reply["result"] = Error.LEGAL reply["nickname"] = self.nickname() reply["fullname"] = self.full_name() sm.populate_dict(reply) return reply
def deferred_ratings(): """ This is the deferred ratings table calculation process """ with Client.get_context() as context: t0 = time.time() try: _create_ratings() except Exception as ex: logging.error("Exception in deferred_ratings: {0!r}".format(ex)) return t1 = time.time() StatsModel.log_cache_stats() # Do not maintain the cache in memory between runs StatsModel.clear_cache() logging.info( "Ratings calculation finished in {0:.2f} seconds".format(t1 - t0))
def _write_stats(timestamp, urecs): """ Writes the freshly calculated statistics records to the database """ # Delete all previous stats with the same timestamp, if any StatsModel.delete_ts(timestamp = timestamp) um_list = [] for sm in urecs.values(): # Set the reference timestamp for the entire stats series sm.timestamp = timestamp # Fetch user information to update Elo statistics if sm.user: # Not robot um = UserModel.fetch(sm.user.id()) if um: um.elo = sm.elo um.human_elo = sm.human_elo um_list.append(um) # Update the statistics records StatsModel.put_multi(urecs.values()) # Update the user records UserModel.put_multi(um_list)
def _write_stats(timestamp, urecs): """ Writes the freshly calculated statistics records to the database """ # Delete all previous stats with the same timestamp, if any StatsModel.delete_ts(timestamp=timestamp) um_list = [] for sm in urecs.values(): # Set the reference timestamp for the entire stats series sm.timestamp = timestamp # Fetch user information to update Elo statistics if sm.user: # Not robot um = UserModel.fetch(sm.user.id()) if um: um.elo = sm.elo um.human_elo = sm.human_elo um_list.append(um) # Update the statistics records StatsModel.put_multi(urecs.values()) # Update the user records UserModel.put_multi(um_list)
def statistics(self): """ Return a set of key statistics on the user """ reply = dict() sm = StatsModel.newest_for_user(self.id()) reply["result"] = Error.LEGAL reply["nickname"] = self.nickname() reply["fullname"] = self.full_name() sm.populate_dict(reply) # Add statistics from the user entity reply["highest_score"] = self._highest_score reply["highest_score_game"] = self._highest_score_game reply["best_word"] = self._best_word reply["best_word_score"] = self._best_word_score reply["best_word_game"] = self._best_word_game return reply
def _create_ratings(timestamp): """ Create the Top 100 ratings tables """ logging.info(u"Starting _create_ratings") _key = StatsModel.dict_key def _augment_table(t, t_yesterday, t_week_ago, t_month_ago): """ Go through a table of top scoring users and augment it with data from previous time points """ for sm in t: # Augment the rating with info about progress key = _key(sm) def _augment(prop): sm[prop + "_yesterday"] = t_yesterday[key][prop] if key in t_yesterday else 0 sm[prop + "_week_ago"] = t_week_ago[key][prop] if key in t_week_ago else 0 sm[prop + "_month_ago"] = t_month_ago[key][prop] if key in t_month_ago else 0 _augment("rank") _augment("games") _augment("elo") _augment("wins") _augment("losses") _augment("score") _augment("score_against") # All players including robot games top100_all = [ sm for sm in StatsModel.list_elo(timestamp, 100) ] top100_all_yesterday = { _key(sm) : sm for sm in StatsModel.list_elo(timestamp - timedelta(days = 1), 100) } top100_all_week_ago = { _key(sm) : sm for sm in StatsModel.list_elo(timestamp - timedelta(days = 7), 100) } top100_all_month_ago = { _key(sm) : sm for sm in StatsModel.list_elo(monthdelta(timestamp, -1), 100) } # Augment the table for all games _augment_table(top100_all, top100_all_yesterday, top100_all_week_ago, top100_all_month_ago) # Human only games top100_human = [ sm for sm in StatsModel.list_human_elo(timestamp, 100) ] top100_human_yesterday = { _key(sm) : sm for sm in StatsModel.list_human_elo(timestamp - timedelta(days = 1), 100) } top100_human_week_ago = { _key(sm) : sm for sm in StatsModel.list_human_elo(timestamp - timedelta(days = 7), 100) } top100_human_month_ago = { _key(sm) : sm for sm in StatsModel.list_human_elo(monthdelta(timestamp, -1), 100) } # Augment the table for human only games _augment_table(top100_human, top100_human_yesterday, top100_human_week_ago, top100_human_month_ago) logging.info(u"Writing top 100 tables to the database") # Write the Top 100 tables to the database for rank in range(0, 100): # All players including robots rm = RatingModel.get_or_create("all", rank + 1) if rank < len(top100_all): rm.assign(top100_all[rank]) else: # Sentinel empty records rm.user = None rm.robot_level = -1 rm.games = -1 rm.put() # Humans only rm = RatingModel.get_or_create("human", rank + 1) if rank < len(top100_human): rm.assign(top100_human[rank]) else: # Sentinel empty records rm.user = None rm.robot_level = -1 rm.games = -1 rm.put() logging.info(u"Finishing _create_ratings")
def _init_stat(user_id, robot_level): """ Returns the newest StatsModel instance available for the given user """ return StatsModel.newest_before(from_time, user_id, robot_level)
def _create_ratings(): """ Create the Top 100 ratings tables """ logging.info(u"Starting _create_ratings") _key = StatsModel.dict_key timestamp = datetime.utcnow() yesterday = timestamp - timedelta(days=1) week_ago = timestamp - timedelta(days=7) month_ago = monthdelta(timestamp, -1) def _augment_table(t, t_yesterday, t_week_ago, t_month_ago): """ Go through a table of top scoring users and augment it with data from previous time points """ for sm in t: # Augment the rating with info about progress key = _key(sm) def _augment(prop): sm[prop + "_yesterday"] = t_yesterday[key][ prop] if key in t_yesterday else 0 sm[prop + "_week_ago"] = t_week_ago[key][ prop] if key in t_week_ago else 0 sm[prop + "_month_ago"] = t_month_ago[key][ prop] if key in t_month_ago else 0 _augment("rank") _augment("games") _augment("elo") _augment("wins") _augment("losses") _augment("score") _augment("score_against") # All players including robot games #top100_all = StatsModel.list_elo(None, 100) top100_all = [sm for sm in StatsModel.list_elo(timestamp, 100)] top100_all_yesterday = { _key(sm): sm for sm in StatsModel.list_elo(yesterday, 100) } top100_all_week_ago = { _key(sm): sm for sm in StatsModel.list_elo(week_ago, 100) } top100_all_month_ago = { _key(sm): sm for sm in StatsModel.list_elo(month_ago, 100) } # Augment the table for all games _augment_table(top100_all, top100_all_yesterday, top100_all_week_ago, top100_all_month_ago) # Human only games #top100_human = StatsModel.list_human_elo(None, 100) top100_human = [sm for sm in StatsModel.list_human_elo(timestamp, 100)] top100_human_yesterday = { _key(sm): sm for sm in StatsModel.list_human_elo(yesterday, 100) } top100_human_week_ago = { _key(sm): sm for sm in StatsModel.list_human_elo(week_ago, 100) } top100_human_month_ago = { _key(sm): sm for sm in StatsModel.list_human_elo(month_ago, 100) } # Augment the table for human only games _augment_table(top100_human, top100_human_yesterday, top100_human_week_ago, top100_human_month_ago) logging.info(u"Writing top 100 tables to the database") # Write the Top 100 tables to the database rlist = [] for rank in range(0, 100): # All players including robots rm = RatingModel.get_or_create("all", rank + 1) if rank < len(top100_all): rm.assign(top100_all[rank]) else: # Sentinel empty records rm.user = None rm.robot_level = -1 rm.games = -1 rlist.append(rm) # Humans only rm = RatingModel.get_or_create("human", rank + 1) if rank < len(top100_human): rm.assign(top100_human[rank]) else: # Sentinel empty records rm.user = None rm.robot_level = -1 rm.games = -1 rlist.append(rm) # Put the entire top 100 table in one RPC call RatingModel.put_multi(rlist) logging.info(u"Finishing _create_ratings")
def _run_stats(from_time, to_time): """ Runs a process to update user statistics and Elo ratings """ logging.info("Generating stats from {0} to {1}".format(from_time, to_time)) if from_time is None or to_time is None: # Time range must be specified return False if from_time >= to_time: # Null time range return False # Clear previous cache contents, if any StatsModel.clear_cache() # Iterate over all finished games within the time span in temporal order # pylint: disable=singleton-comparison q = (GameModel.query( ndb.AND(GameModel.ts_last_move > from_time, GameModel.ts_last_move <= to_time)).order( GameModel.ts_last_move).filter(GameModel.over == True)) # The accumulated user statistics users = dict() def _init_stat(user_id, robot_level): """ Returns the newest StatsModel instance available for the given user """ return StatsModel.newest_before(from_time, user_id, robot_level) cnt = 0 ts_last_processed = None try: # Use i as a progress counter i = 0 for gm in iter_q(q, chunk_size=250): i += 1 lm = Alphabet.format_timestamp(gm.ts_last_move or gm.timestamp) p0 = None if gm.player0 is None else gm.player0.id() p1 = None if gm.player1 is None else gm.player1.id() robot_game = (p0 is None) or (p1 is None) if robot_game: rl = gm.robot_level else: rl = 0 s0 = gm.score0 s1 = gm.score1 if (s0 == 0) and (s1 == 0): # When a game ends by resigning immediately, # make sure that the weaker player # doesn't get Elo points for a draw; in fact, # ignore such a game altogether in the statistics continue if p0 is None: k0 = "robot-" + str(rl) else: k0 = p0 if p1 is None: k1 = "robot-" + str(rl) else: k1 = p1 if k0 in users: urec0 = users[k0] else: users[k0] = urec0 = _init_stat(p0, rl if p0 is None else 0) if k1 in users: urec1 = users[k1] else: users[k1] = urec1 = _init_stat(p1, rl if p1 is None else 0) # Number of games played urec0.games += 1 urec1.games += 1 if not robot_game: urec0.human_games += 1 urec1.human_games += 1 # Total scores urec0.score += s0 urec1.score += s1 urec0.score_against += s1 urec1.score_against += s0 if not robot_game: urec0.human_score += s0 urec1.human_score += s1 urec0.human_score_against += s1 urec1.human_score_against += s0 # Wins and losses if s0 > s1: urec0.wins += 1 urec1.losses += 1 elif s1 > s0: urec1.wins += 1 urec0.losses += 1 if not robot_game: if s0 > s1: urec0.human_wins += 1 urec1.human_losses += 1 elif s1 > s0: urec1.human_wins += 1 urec0.human_losses += 1 # Find out whether players are established or beginners est0 = urec0.games > ESTABLISHED_MARK est1 = urec1.games > ESTABLISHED_MARK # Save the Elo point state used in the calculation gm.elo0, gm.elo1 = urec0.elo, urec1.elo # Compute the Elo points of both players adj = _compute_elo((urec0.elo, urec1.elo), s0, s1, est0, est1) # When an established player is playing a beginning (provisional) player, # leave the Elo score of the established player unchanged # Adjust player 0 if est0 and not est1: adj = (0, adj[1]) gm.elo0_adj = adj[0] urec0.elo += adj[0] # Adjust player 1 if est1 and not est0: adj = (adj[0], 0) gm.elo1_adj = adj[1] urec1.elo += adj[1] # If not a robot game, compute the human-only Elo if not robot_game: gm.human_elo0, gm.human_elo1 = urec0.human_elo, urec1.human_elo adj = _compute_elo((urec0.human_elo, urec1.human_elo), s0, s1, est0, est1) # Adjust player 0 if est0 and not est1: adj = (0, adj[1]) gm.human_elo0_adj = adj[0] urec0.human_elo += adj[0] # Adjust player 1 if est1 and not est0: adj = (adj[0], 0) gm.human_elo1_adj = adj[1] urec1.human_elo += adj[1] # Save the game object with the new Elo adjustment statistics gm.put() # Save the last processed timestamp ts_last_processed = lm cnt += 1 # Report on our progress if i % 500 == 0: logging.info("Processed {0} games".format(i)) except Exception as ex: logging.error( "Exception in _run_stats() after {0} games and {1} users: {2!r}". format(cnt, len(users), ex)) return False # Completed without incident logging.info( "Normal completion of stats for {1} games and {0} users".format( len(users), cnt)) _write_stats(to_time, users) return True