def deferred_ratings():
""" This is the deferred ratings table calculation process """
# Disable the in-context cache to save memory
# (it doesn't give any speed advantage for this processing)
Context.disable_cache()
t0 = time.time()
try:
_create_ratings()
except DeadlineExceededError as ex:
# Hit deadline: save the stuff we already have and
# defer a new task to continue where we left off
logging.error(u"Deadline exceeded in ratings, failing permamently")
# Normal return prevents this task from being run again
raise deferred.PermanentTaskFailure()
except Exception as ex:
logging.error(u"Exception in ratings, failing permanently: {0}".format(ex))
# Avoid having the task retried
raise deferred.PermanentTaskFailure()
t1 = time.time()
logging.info(u"Ratings calculation finished in {0:.2f} seconds".format(t1 - t0))
StatsModel.log_cache_stats()
StatsModel.clear_cache() # Do not maintain the cache in memory between runs
def deferred_ratings():
""" This is the deferred ratings table calculation process """
# Disable the in-context cache to save memory
# (it doesn't give any speed advantage for this processing)
Context.disable_cache()
t0 = time.time()
try:
_create_ratings()
except DeadlineExceededError as ex:
# Hit deadline: save the stuff we already have and
# defer a new task to continue where we left off
logging.error(u"Deadline exceeded in ratings, failing permamently")
# Normal return prevents this task from being run again
raise deferred.PermanentTaskFailure()
except Exception as ex:
logging.error(
u"Exception in ratings, failing permanently: {0}".format(ex))
# Avoid having the task retried
raise deferred.PermanentTaskFailure()
t1 = time.time()
logging.info(u"Ratings calculation finished in {0:.2f} seconds".format(t1 -
t0))
StatsModel.log_cache_stats()
StatsModel.clear_cache(
) # Do not maintain the cache in memory between runs
def statistics(self):
""" Return a set of key statistics on the user """
reply = dict()
sm = StatsModel.newest_for_user(self.id())
reply["result"] = Error.LEGAL
reply["nickname"] = self.nickname()
reply["fullname"] = self.full_name()
sm.populate_dict(reply)
return reply
def deferred_ratings():
""" This is the deferred ratings table calculation process """
with Client.get_context() as context:
t0 = time.time()
try:
_create_ratings()
except Exception as ex:
logging.error("Exception in deferred_ratings: {0!r}".format(ex))
return
t1 = time.time()
StatsModel.log_cache_stats()
# Do not maintain the cache in memory between runs
StatsModel.clear_cache()
logging.info(
"Ratings calculation finished in {0:.2f} seconds".format(t1 - t0))
def _write_stats(timestamp, urecs):
""" Writes the freshly calculated statistics records to the database """
# Delete all previous stats with the same timestamp, if any
StatsModel.delete_ts(timestamp = timestamp)
um_list = []
for sm in urecs.values():
# Set the reference timestamp for the entire stats series
sm.timestamp = timestamp
# Fetch user information to update Elo statistics
if sm.user:
# Not robot
um = UserModel.fetch(sm.user.id())
if um:
um.elo = sm.elo
um.human_elo = sm.human_elo
um_list.append(um)
# Update the statistics records
StatsModel.put_multi(urecs.values())
# Update the user records
UserModel.put_multi(um_list)
def _write_stats(timestamp, urecs):
""" Writes the freshly calculated statistics records to the database """
# Delete all previous stats with the same timestamp, if any
StatsModel.delete_ts(timestamp=timestamp)
um_list = []
for sm in urecs.values():
# Set the reference timestamp for the entire stats series
sm.timestamp = timestamp
# Fetch user information to update Elo statistics
if sm.user:
# Not robot
um = UserModel.fetch(sm.user.id())
if um:
um.elo = sm.elo
um.human_elo = sm.human_elo
um_list.append(um)
# Update the statistics records
StatsModel.put_multi(urecs.values())
# Update the user records
UserModel.put_multi(um_list)
def statistics(self):
""" Return a set of key statistics on the user """
reply = dict()
sm = StatsModel.newest_for_user(self.id())
reply["result"] = Error.LEGAL
reply["nickname"] = self.nickname()
reply["fullname"] = self.full_name()
sm.populate_dict(reply)
# Add statistics from the user entity
reply["highest_score"] = self._highest_score
reply["highest_score_game"] = self._highest_score_game
reply["best_word"] = self._best_word
reply["best_word_score"] = self._best_word_score
reply["best_word_game"] = self._best_word_game
return reply
def statistics(self):
""" Return a set of key statistics on the user """
reply = dict()
sm = StatsModel.newest_for_user(self.id())
reply["result"] = Error.LEGAL
reply["nickname"] = self.nickname()
reply["fullname"] = self.full_name()
sm.populate_dict(reply)
# Add statistics from the user entity
reply["highest_score"] = self._highest_score
reply["highest_score_game"] = self._highest_score_game
reply["best_word"] = self._best_word
reply["best_word_score"] = self._best_word_score
reply["best_word_game"] = self._best_word_game
return reply
def _create_ratings(timestamp):
""" Create the Top 100 ratings tables """
logging.info(u"Starting _create_ratings")
_key = StatsModel.dict_key
def _augment_table(t, t_yesterday, t_week_ago, t_month_ago):
""" Go through a table of top scoring users and augment it with data from previous time points """
for sm in t:
# Augment the rating with info about progress
key = _key(sm)
def _augment(prop):
sm[prop + "_yesterday"] = t_yesterday[key][prop] if key in t_yesterday else 0
sm[prop + "_week_ago"] = t_week_ago[key][prop] if key in t_week_ago else 0
sm[prop + "_month_ago"] = t_month_ago[key][prop] if key in t_month_ago else 0
_augment("rank")
_augment("games")
_augment("elo")
_augment("wins")
_augment("losses")
_augment("score")
_augment("score_against")
# All players including robot games
top100_all = [ sm for sm in StatsModel.list_elo(timestamp, 100) ]
top100_all_yesterday = { _key(sm) : sm for sm in StatsModel.list_elo(timestamp - timedelta(days = 1), 100) }
top100_all_week_ago = { _key(sm) : sm for sm in StatsModel.list_elo(timestamp - timedelta(days = 7), 100) }
top100_all_month_ago = { _key(sm) : sm for sm in StatsModel.list_elo(monthdelta(timestamp, -1), 100) }
# Augment the table for all games
_augment_table(top100_all, top100_all_yesterday, top100_all_week_ago, top100_all_month_ago)
# Human only games
top100_human = [ sm for sm in StatsModel.list_human_elo(timestamp, 100) ]
top100_human_yesterday = { _key(sm) : sm for sm in StatsModel.list_human_elo(timestamp - timedelta(days = 1), 100) }
top100_human_week_ago = { _key(sm) : sm for sm in StatsModel.list_human_elo(timestamp - timedelta(days = 7), 100) }
top100_human_month_ago = { _key(sm) : sm for sm in StatsModel.list_human_elo(monthdelta(timestamp, -1), 100) }
# Augment the table for human only games
_augment_table(top100_human, top100_human_yesterday, top100_human_week_ago, top100_human_month_ago)
logging.info(u"Writing top 100 tables to the database")
# Write the Top 100 tables to the database
for rank in range(0, 100):
# All players including robots
rm = RatingModel.get_or_create("all", rank + 1)
if rank < len(top100_all):
rm.assign(top100_all[rank])
else:
# Sentinel empty records
rm.user = None
rm.robot_level = -1
rm.games = -1
rm.put()
# Humans only
rm = RatingModel.get_or_create("human", rank + 1)
if rank < len(top100_human):
rm.assign(top100_human[rank])
else:
# Sentinel empty records
rm.user = None
rm.robot_level = -1
rm.games = -1
rm.put()
logging.info(u"Finishing _create_ratings")
def _init_stat(user_id, robot_level):
""" Returns the newest StatsModel instance available for the given user """
return StatsModel.newest_before(from_time, user_id, robot_level)
def _create_ratings():
""" Create the Top 100 ratings tables """
logging.info(u"Starting _create_ratings")
_key = StatsModel.dict_key
timestamp = datetime.utcnow()
yesterday = timestamp - timedelta(days=1)
week_ago = timestamp - timedelta(days=7)
month_ago = monthdelta(timestamp, -1)
def _augment_table(t, t_yesterday, t_week_ago, t_month_ago):
""" Go through a table of top scoring users and augment it with data from previous time points """
for sm in t:
# Augment the rating with info about progress
key = _key(sm)
def _augment(prop):
sm[prop + "_yesterday"] = t_yesterday[key][
prop] if key in t_yesterday else 0
sm[prop + "_week_ago"] = t_week_ago[key][
prop] if key in t_week_ago else 0
sm[prop + "_month_ago"] = t_month_ago[key][
prop] if key in t_month_ago else 0
_augment("rank")
_augment("games")
_augment("elo")
_augment("wins")
_augment("losses")
_augment("score")
_augment("score_against")
# All players including robot games
#top100_all = StatsModel.list_elo(None, 100)
top100_all = [sm for sm in StatsModel.list_elo(timestamp, 100)]
top100_all_yesterday = {
_key(sm): sm
for sm in StatsModel.list_elo(yesterday, 100)
}
top100_all_week_ago = {
_key(sm): sm
for sm in StatsModel.list_elo(week_ago, 100)
}
top100_all_month_ago = {
_key(sm): sm
for sm in StatsModel.list_elo(month_ago, 100)
}
# Augment the table for all games
_augment_table(top100_all, top100_all_yesterday, top100_all_week_ago,
top100_all_month_ago)
# Human only games
#top100_human = StatsModel.list_human_elo(None, 100)
top100_human = [sm for sm in StatsModel.list_human_elo(timestamp, 100)]
top100_human_yesterday = {
_key(sm): sm
for sm in StatsModel.list_human_elo(yesterday, 100)
}
top100_human_week_ago = {
_key(sm): sm
for sm in StatsModel.list_human_elo(week_ago, 100)
}
top100_human_month_ago = {
_key(sm): sm
for sm in StatsModel.list_human_elo(month_ago, 100)
}
# Augment the table for human only games
_augment_table(top100_human, top100_human_yesterday, top100_human_week_ago,
top100_human_month_ago)
logging.info(u"Writing top 100 tables to the database")
# Write the Top 100 tables to the database
rlist = []
for rank in range(0, 100):
# All players including robots
rm = RatingModel.get_or_create("all", rank + 1)
if rank < len(top100_all):
rm.assign(top100_all[rank])
else:
# Sentinel empty records
rm.user = None
rm.robot_level = -1
rm.games = -1
rlist.append(rm)
# Humans only
rm = RatingModel.get_or_create("human", rank + 1)
if rank < len(top100_human):
rm.assign(top100_human[rank])
else:
# Sentinel empty records
rm.user = None
rm.robot_level = -1
rm.games = -1
rlist.append(rm)
# Put the entire top 100 table in one RPC call
RatingModel.put_multi(rlist)
logging.info(u"Finishing _create_ratings")
def _init_stat(user_id, robot_level):
""" Returns the newest StatsModel instance available for the given user """
return StatsModel.newest_before(from_time, user_id, robot_level)
def _run_stats(from_time, to_time):
""" Runs a process to update user statistics and Elo ratings """
logging.info("Generating stats from {0} to {1}".format(from_time, to_time))
if from_time is None or to_time is None:
# Time range must be specified
return False
if from_time >= to_time:
# Null time range
return False
# Clear previous cache contents, if any
StatsModel.clear_cache()
# Iterate over all finished games within the time span in temporal order
# pylint: disable=singleton-comparison
q = (GameModel.query(
ndb.AND(GameModel.ts_last_move > from_time,
GameModel.ts_last_move <= to_time)).order(
GameModel.ts_last_move).filter(GameModel.over == True))
# The accumulated user statistics
users = dict()
def _init_stat(user_id, robot_level):
""" Returns the newest StatsModel instance available for the given user """
return StatsModel.newest_before(from_time, user_id, robot_level)
cnt = 0
ts_last_processed = None
try:
# Use i as a progress counter
i = 0
for gm in iter_q(q, chunk_size=250):
i += 1
lm = Alphabet.format_timestamp(gm.ts_last_move or gm.timestamp)
p0 = None if gm.player0 is None else gm.player0.id()
p1 = None if gm.player1 is None else gm.player1.id()
robot_game = (p0 is None) or (p1 is None)
if robot_game:
rl = gm.robot_level
else:
rl = 0
s0 = gm.score0
s1 = gm.score1
if (s0 == 0) and (s1 == 0):
# When a game ends by resigning immediately,
# make sure that the weaker player
# doesn't get Elo points for a draw; in fact,
# ignore such a game altogether in the statistics
continue
if p0 is None:
k0 = "robot-" + str(rl)
else:
k0 = p0
if p1 is None:
k1 = "robot-" + str(rl)
else:
k1 = p1
if k0 in users:
urec0 = users[k0]
else:
users[k0] = urec0 = _init_stat(p0, rl if p0 is None else 0)
if k1 in users:
urec1 = users[k1]
else:
users[k1] = urec1 = _init_stat(p1, rl if p1 is None else 0)
# Number of games played
urec0.games += 1
urec1.games += 1
if not robot_game:
urec0.human_games += 1
urec1.human_games += 1
# Total scores
urec0.score += s0
urec1.score += s1
urec0.score_against += s1
urec1.score_against += s0
if not robot_game:
urec0.human_score += s0
urec1.human_score += s1
urec0.human_score_against += s1
urec1.human_score_against += s0
# Wins and losses
if s0 > s1:
urec0.wins += 1
urec1.losses += 1
elif s1 > s0:
urec1.wins += 1
urec0.losses += 1
if not robot_game:
if s0 > s1:
urec0.human_wins += 1
urec1.human_losses += 1
elif s1 > s0:
urec1.human_wins += 1
urec0.human_losses += 1
# Find out whether players are established or beginners
est0 = urec0.games > ESTABLISHED_MARK
est1 = urec1.games > ESTABLISHED_MARK
# Save the Elo point state used in the calculation
gm.elo0, gm.elo1 = urec0.elo, urec1.elo
# Compute the Elo points of both players
adj = _compute_elo((urec0.elo, urec1.elo), s0, s1, est0, est1)
# When an established player is playing a beginning (provisional) player,
# leave the Elo score of the established player unchanged
# Adjust player 0
if est0 and not est1:
adj = (0, adj[1])
gm.elo0_adj = adj[0]
urec0.elo += adj[0]
# Adjust player 1
if est1 and not est0:
adj = (adj[0], 0)
gm.elo1_adj = adj[1]
urec1.elo += adj[1]
# If not a robot game, compute the human-only Elo
if not robot_game:
gm.human_elo0, gm.human_elo1 = urec0.human_elo, urec1.human_elo
adj = _compute_elo((urec0.human_elo, urec1.human_elo), s0, s1,
est0, est1)
# Adjust player 0
if est0 and not est1:
adj = (0, adj[1])
gm.human_elo0_adj = adj[0]
urec0.human_elo += adj[0]
# Adjust player 1
if est1 and not est0:
adj = (adj[0], 0)
gm.human_elo1_adj = adj[1]
urec1.human_elo += adj[1]
# Save the game object with the new Elo adjustment statistics
gm.put()
# Save the last processed timestamp
ts_last_processed = lm
cnt += 1
# Report on our progress
if i % 500 == 0:
logging.info("Processed {0} games".format(i))
except Exception as ex:
logging.error(
"Exception in _run_stats() after {0} games and {1} users: {2!r}".
format(cnt, len(users), ex))
return False
# Completed without incident
logging.info(
"Normal completion of stats for {1} games and {0} users".format(
len(users), cnt))
_write_stats(to_time, users)
return True