class Tournament(ABC):
"""Abstract base class for Arena/Swisss/RR Tournament classes
They have to implement create_pairing() for waiting_players"""
system: ClassVar[int] = ARENA
def __init__(
self,
app,
tournamentId,
variant="chess",
chess960=False,
rated=True,
before_start=5,
minutes=45,
name="",
description="",
fen="",
base=1,
inc=0,
byoyomi_period=0,
rounds=0,
created_by="",
created_at=None,
starts_at=None,
status=None,
with_clock=True,
frequency="",
):
self.app = app
self.id = tournamentId
self.name = name
self.description = description
self.variant = variant
self.rated = rated
self.before_start = before_start # in minutes
self.minutes = minutes # in minutes
self.fen = fen
self.base = base
self.inc = inc
self.byoyomi_period = byoyomi_period
self.chess960 = chess960
self.rounds = rounds
self.frequency = frequency
self.created_by = created_by
self.created_at = datetime.now(timezone.utc) if created_at is None else created_at
if starts_at == "" or starts_at is None:
self.starts_at = self.created_at + timedelta(seconds=int(before_start * 60))
else:
self.starts_at = starts_at
# TODO: calculate wave from TC, variant, number of players
self.wave = timedelta(seconds=3)
self.wave_delta = timedelta(seconds=1)
self.current_round = 0
self.prev_pairing = None
self.messages = collections.deque([], MAX_CHAT_LINES)
self.spectators = set()
self.players: dict[User, PlayerData] = {}
self.leaderboard = ValueSortedDict(neg)
self.leaderboard_keys_view = SortedKeysView(self.leaderboard)
self.status = T_CREATED if status is None else status
self.ongoing_games = 0
self.nb_players = 0
self.nb_games_finished = 0
self.w_win = 0
self.b_win = 0
self.draw = 0
self.nb_berserk = 0
self.nb_games_cached = -1
self.leaderboard_cache = {}
self.first_pairing = False
self.top_player = None
self.top_game = None
self.notify1 = False
self.notify2 = False
if minutes is None:
self.ends_at = self.starts_at + timedelta(days=1)
else:
self.ends_at = self.starts_at + timedelta(minutes=minutes)
if with_clock:
self.clock_task = asyncio.create_task(self.clock())
def __repr__(self):
return " ".join((self.id, self.name, self.created_at.isoformat()))
@abstractmethod
def create_pairing(self, waiting_players):
pass
def user_status(self, user):
if user in self.players:
return (
"paused"
if self.players[user].paused
else "withdrawn"
if self.players[user].withdrawn
else "joined"
)
else:
return "spectator"
def user_rating(self, user):
if user in self.players:
return self.players[user].rating
else:
return "%s%s" % user.get_rating(self.variant, self.chess960).rating_prov
def players_json(self, page=None, user=None):
if (page is None) and (user is not None) and (user in self.players):
if self.players[user].page > 0:
page = self.players[user].page
else:
div, mod = divmod(self.leaderboard.index(user) + 1, 10)
page = div + (1 if mod > 0 else 0)
if self.status == T_CREATED:
self.players[user].page = page
if page is None:
page = 1
if self.nb_games_cached != self.nb_games_finished:
# number of games changed (game ended)
self.leaderboard_cache = {}
self.nb_games_cached = self.nb_games_finished
elif user is not None:
if self.status == T_STARTED:
# player status changed (JOIN/PAUSE)
if page in self.leaderboard_cache:
del self.leaderboard_cache[page]
elif self.status == T_CREATED:
# number of players changed (JOIN/WITHDRAW)
self.leaderboard_cache = {}
if page in self.leaderboard_cache:
return self.leaderboard_cache[page]
def player_json(player, full_score):
return {
"paused": self.players[player].paused if self.status == T_STARTED else False,
"title": player.title,
"name": player.username,
"rating": self.players[player].rating,
"points": self.players[player].points,
"fire": self.players[player].win_streak,
"score": full_score, # SCORE_SHIFT-ed + performance rating
"perf": self.players[player].performance,
"nbGames": self.players[player].nb_games,
"nbWin": self.players[player].nb_win,
"nbBerserk": self.players[player].nb_berserk,
}
start = (page - 1) * 10
end = min(start + 10, self.nb_players)
page_json = {
"type": "get_players",
"requestedBy": user.username if user is not None else "",
"nbPlayers": self.nb_players,
"nbGames": self.nb_games_finished,
"page": page,
"players": [
player_json(player, full_score)
for player, full_score in self.leaderboard.items()[start:end]
],
}
if self.status > T_STARTED:
page_json["podium"] = [
player_json(player, full_score)
for player, full_score in self.leaderboard.items()[0:3]
]
self.leaderboard_cache[page] = page_json
return page_json
# TODO: cache this
def games_json(self, player_name):
player = self.app["users"].get(player_name)
return {
"type": "get_games",
"rank": self.leaderboard.index(player) + 1,
"title": player.title,
"name": player_name,
"perf": self.players[player].performance,
"nbGames": self.players[player].nb_games,
"nbWin": self.players[player].nb_win,
"nbBerserk": self.players[player].nb_berserk,
"games": [game.game_json(player) for game in self.players[player].games],
}
@property
def spectator_list(self):
return spectators(self)
@property
def top_game_json(self):
return {
"type": "top_game",
"gameId": self.top_game.id,
"variant": self.top_game.variant,
"fen": self.top_game.board.fen,
"w": self.top_game.wplayer.username,
"b": self.top_game.bplayer.username,
"wr": self.leaderboard_keys_view.index(self.top_game.wplayer) + 1,
"br": self.leaderboard_keys_view.index(self.top_game.bplayer) + 1,
"chess960": self.top_game.chess960,
"base": self.top_game.base,
"inc": self.top_game.inc,
"byoyomi": self.top_game.byoyomi_period,
}
def waiting_players(self):
return [
p
for p in self.leaderboard
if self.players[p].free
and self.id in p.tournament_sockets
and len(p.tournament_sockets[self.id]) > 0
and not self.players[p].paused
and not self.players[p].withdrawn
]
async def clock(self):
try:
while self.status not in (T_ABORTED, T_FINISHED, T_ARCHIVED):
now = datetime.now(timezone.utc)
if self.status == T_CREATED:
remaining_time = self.starts_at - now
remaining_mins_to_start = int(
((remaining_time.days * 3600 * 24) + remaining_time.seconds) / 60
)
if now >= self.starts_at:
if self.system != ARENA and len(self.players) < 3:
# Swiss and RR Tournaments need at least 3 players to start
await self.abort()
print("T_ABORTED: less than 3 player joined")
break
await self.start(now)
continue
elif (not self.notify2) and remaining_mins_to_start <= NOTIFY2_MINUTES:
self.notify1 = True
self.notify2 = True
await discord_message(
self.app,
"notify_tournament",
self.notify_discord_msg(remaining_mins_to_start),
)
continue
elif (not self.notify1) and remaining_mins_to_start <= NOTIFY1_MINUTES:
self.notify1 = True
await discord_message(
self.app,
"notify_tournament",
self.notify_discord_msg(remaining_mins_to_start),
)
continue
elif (self.minutes is not None) and now >= self.ends_at:
await self.finish()
print("T_FINISHED: no more time left")
break
elif self.status == T_STARTED:
if self.system == ARENA:
# In case of server restart
if self.prev_pairing is None:
self.prev_pairing = now - self.wave
if now >= self.prev_pairing + self.wave + random.uniform(
-self.wave_delta, self.wave_delta
):
waiting_players = self.waiting_players()
nb_waiting_players = len(waiting_players)
if nb_waiting_players >= 2:
log.debug("Enough player (%s), do pairing", nb_waiting_players)
await self.create_new_pairings(waiting_players)
self.prev_pairing = now
else:
log.debug(
"Too few player (%s) to make pairing",
nb_waiting_players,
)
else:
log.debug("Waiting for new pairing wave...")
elif self.ongoing_games == 0:
if self.current_round < self.rounds:
self.current_round += 1
log.debug("Do %s. round pairing", self.current_round)
waiting_players = self.waiting_players()
await self.create_new_pairings(waiting_players)
else:
await self.finish()
log.debug("T_FINISHED: no more round left")
break
else:
print(
"%s has %s ongoing game(s)..."
% (
"RR" if self.system == RR else "Swiss",
self.ongoing_games,
)
)
log.debug("%s CLOCK %s", self.id, now.strftime("%H:%M:%S"))
await asyncio.sleep(1)
except Exception:
log.exception("Exception in tournament clock()")
async def start(self, now):
self.status = T_STARTED
self.first_pairing = True
self.set_top_player()
response = {
"type": "tstatus",
"tstatus": self.status,
"secondsToFinish": (self.ends_at - now).total_seconds(),
}
await self.broadcast(response)
# force first pairing wave in arena
if self.system == ARENA:
self.prev_pairing = now - self.wave
if self.app["db"] is not None:
print(
await self.app["db"].tournament.find_one_and_update(
{"_id": self.id},
{"$set": {"status": self.status}},
return_document=ReturnDocument.AFTER,
)
)
@property
def summary(self):
return {
"type": "tstatus",
"tstatus": self.status,
"nbPlayers": self.nb_players,
"nbGames": self.nb_games_finished,
"wWin": self.w_win,
"bWin": self.b_win,
"draw": self.draw,
"berserk": self.nb_berserk,
"sumRating": sum(
self.players[player].rating
for player in self.players
if not self.players[player].withdrawn
),
}
async def finalize(self, status):
self.status = status
if len(self.players) > 0:
self.print_leaderboard()
print("--- TOURNAMENT RESULT ---")
for i in range(min(3, len(self.leaderboard))):
player = self.leaderboard.peekitem(i)[0]
print("--- #%s ---" % (i + 1), player.username)
# remove latest games from players tournament if it was not finished in time
for player in self.players:
if len(self.players[player].games) == 0:
continue
latest = self.players[player].games[-1]
if latest and latest.status in (CREATED, STARTED):
self.players[player].games.pop()
self.players[player].points.pop()
self.players[player].nb_games -= 1
# force to create new players json data
self.nb_games_cached = -1
await self.broadcast(self.summary)
await self.save()
await self.broadcast_spotlight()
async def broadcast_spotlight(self):
spotlights = tournament_spotlights(self.app["tournaments"])
lobby_sockets = self.app["lobbysockets"]
response = {"type": "spotlights", "items": spotlights}
await lobby_broadcast(lobby_sockets, response)
async def abort(self):
await self.finalize(T_ABORTED)
async def finish(self):
await self.finalize(T_FINISHED)
async def join(self, user):
if user.anon:
return
if self.system == RR and len(self.players) > self.rounds + 1:
raise EnoughPlayer
if user not in self.players:
# new player joined
rating, provisional = user.get_rating(self.variant, self.chess960).rating_prov
self.players[user] = PlayerData(rating, provisional)
elif self.players[user].withdrawn:
# withdrawn player joined again
rating, provisional = user.get_rating(self.variant, self.chess960).rating_prov
if user not in self.leaderboard:
# new player joined or withdrawn player joined again
if self.status == T_CREATED:
self.leaderboard.setdefault(user, rating)
else:
self.leaderboard.setdefault(user, 0)
self.nb_players += 1
self.players[user].paused = False
self.players[user].withdrawn = False
response = self.players_json(user=user)
await self.broadcast(response)
if self.status == T_CREATED:
await self.broadcast_spotlight()
await self.db_update_player(user, self.players[user])
async def withdraw(self, user):
self.players[user].withdrawn = True
self.leaderboard.pop(user)
self.nb_players -= 1
response = self.players_json(user=user)
await self.broadcast(response)
await self.broadcast_spotlight()
await self.db_update_player(user, self.players[user])
async def pause(self, user):
self.players[user].paused = True
# pause is different from withdraw and join because pause can be initiated from finished games page as well
response = self.players_json(user=user)
await self.broadcast(response)
if (self.top_player is not None) and self.top_player.username == user.username:
self.set_top_player()
await self.db_update_player(user, self.players[user])
def spactator_join(self, spectator):
self.spectators.add(spectator)
def spactator_leave(self, spectator):
self.spectators.discard(spectator)
async def create_new_pairings(self, waiting_players):
pairing = self.create_pairing(waiting_players)
if self.first_pairing:
self.first_pairing = False
# Before tournament starts leaderboard is ordered by ratings
# After first pairing it will be sorted by score points and performance
# so we have to make a clear (all 0) leaderboard here
new_leaderboard = [(user, 0) for user in self.leaderboard]
self.leaderboard = ValueSortedDict(neg, new_leaderboard)
self.leaderboard_keys_view = SortedKeysView(self.leaderboard)
games = await self.create_games(pairing)
return (pairing, games)
def set_top_player(self):
idx = 0
self.top_player = None
while idx < self.nb_players:
top_player = self.leaderboard.peekitem(idx)[0]
if self.players[top_player].paused:
idx += 1
continue
else:
self.top_player = top_player
break
async def create_games(self, pairing):
check_top_game = self.top_player is not None
new_top_game = False
games = []
game_table = None if self.app["db"] is None else self.app["db"].game
for wp, bp in pairing:
game_id = await new_id(game_table)
game = Game(
self.app,
game_id,
self.variant,
self.fen,
wp,
bp,
base=self.base,
inc=self.inc,
byoyomi_period=self.byoyomi_period,
rated=RATED if self.rated else CASUAL,
tournamentId=self.id,
chess960=self.chess960,
)
games.append(game)
self.app["games"][game_id] = game
await insert_game_to_db(game, self.app)
# TODO: save new game to db
if 0: # self.app["db"] is not None:
doc = {
"_id": game.id,
"tid": self.id,
"u": [game.wplayer.username, game.bplayer.username],
"r": "*",
"d": game.date,
"wr": game.wrating,
"br": game.brating,
}
await self.app["db"].tournament_pairing.insert_one(doc)
self.players[wp].games.append(game)
self.players[bp].games.append(game)
self.players[wp].points.append("*")
self.players[bp].points.append("*")
self.ongoing_games += 1
self.players[wp].free = False
self.players[bp].free = False
self.players[wp].nb_games += 1
self.players[bp].nb_games += 1
self.players[wp].prev_opp = game.bplayer.username
self.players[bp].prev_opp = game.wplayer.username
self.players[wp].color_balance += 1
self.players[bp].color_balance -= 1
self.players[wp].nb_not_paired = 0
self.players[bp].nb_not_paired = 0
response = {
"type": "new_game",
"gameId": game_id,
"wplayer": wp.username,
"bplayer": bp.username,
}
try:
ws = next(iter(wp.tournament_sockets[self.id]))
if ws is not None:
await ws.send_json(response)
except Exception:
self.pause(wp)
log.debug("White player %s left the tournament", wp.username)
try:
ws = next(iter(bp.tournament_sockets[self.id]))
if ws is not None:
await ws.send_json(response)
except Exception:
self.pause(bp)
log.debug("Black player %s left the tournament", bp.username)
if (
check_top_game
and (self.top_player is not None)
and self.top_player.username in (game.wplayer.username, game.bplayer.username)
and game.status != BYEGAME
): # Bye game
self.top_game = game
check_top_game = False
new_top_game = True
if new_top_game:
tgj = self.top_game_json
await self.broadcast(tgj)
return games
def points_perfs(self, game: Game) -> Tuple[Point, Point, int, int]:
wplayer = self.players[game.wplayer]
bplayer = self.players[game.bplayer]
wpoint = (0, SCORE)
bpoint = (0, SCORE)
wperf = game.black_rating.rating_prov[0]
bperf = game.white_rating.rating_prov[0]
if game.result == "1/2-1/2":
if self.system == ARENA:
if game.board.ply > 10:
wpoint = (2, SCORE) if wplayer.win_streak == 2 else (1, SCORE)
bpoint = (2, SCORE) if bplayer.win_streak == 2 else (1, SCORE)
wplayer.win_streak = 0
bplayer.win_streak = 0
else:
wpoint, bpoint = (1, SCORE), (1, SCORE)
elif game.result == "1-0":
wplayer.nb_win += 1
if self.system == ARENA:
if wplayer.win_streak == 2:
wpoint = (4, DOUBLE)
else:
wplayer.win_streak += 1
wpoint = (2, STREAK if wplayer.win_streak == 2 else SCORE)
bplayer.win_streak = 0
else:
wpoint = (2, SCORE)
if game.wberserk and game.board.ply >= 13:
wpoint = (wpoint[0] + 1, wpoint[1])
wperf += 500
bperf -= 500
elif game.result == "0-1":
bplayer.nb_win += 1
if self.system == ARENA:
if bplayer.win_streak == 2:
bpoint = (4, DOUBLE)
else:
bplayer.win_streak += 1
bpoint = (2, STREAK if bplayer.win_streak == 2 else SCORE)
wplayer.win_streak = 0
else:
bpoint = (2, SCORE)
if game.bberserk and game.board.ply >= 14:
bpoint = (bpoint[0] + 1, bpoint[1])
wperf -= 500
bperf += 500
return (wpoint, bpoint, wperf, bperf)
def points_perfs_janggi(self, game):
wplayer = self.players[game.wplayer]
bplayer = self.players[game.bplayer]
wpoint = (0, SCORE)
bpoint = (0, SCORE)
wperf = game.black_rating.rating_prov[0]
bperf = game.white_rating.rating_prov[0]
if game.status == VARIANTEND:
wplayer.win_streak = 0
bplayer.win_streak = 0
if game.result == "1-0":
if self.system == ARENA:
wpoint = (4 * 2 if wplayer.win_streak == 2 else 4, SCORE)
bpoint = (2 * 2 if bplayer.win_streak == 2 else 2, SCORE)
else:
wpoint = (4, SCORE)
bpoint = (2, SCORE)
elif game.result == "0-1":
if self.system == ARENA:
bpoint = (4 * 2 if bplayer.win_streak == 2 else 4, SCORE)
wpoint = (2 * 2 if wplayer.win_streak == 2 else 2, SCORE)
else:
bpoint = (4, SCORE)
wpoint = (2, SCORE)
elif game.result == "1-0":
wplayer.nb_win += 1
if self.system == ARENA:
if wplayer.win_streak == 2:
wpoint = (7 * 2, DOUBLE)
else:
wplayer.win_streak += 1
wpoint = (7, STREAK if wplayer.win_streak == 2 else SCORE)
bplayer.win_streak = 0
if game.wberserk and game.board.ply >= 13:
wpoint = (wpoint[0] + 3, wpoint[1])
else:
wpoint = (7, SCORE)
bpoint = (0, SCORE)
wperf += 500
bperf -= 500
elif game.result == "0-1":
bplayer.nb_win += 1
if self.system == ARENA:
if bplayer.win_streak == 2:
bpoint = (7 * 2, DOUBLE)
else:
bplayer.win_streak += 1
bpoint = (7, STREAK if bplayer.win_streak == 2 else SCORE)
wplayer.win_streak = 0
if game.bberserk and game.board.ply >= 14:
bpoint = (bpoint[0] + 3, bpoint[1])
else:
wpoint = (0, SCORE)
bpoint = (7, SCORE)
wperf -= 500
bperf += 500
return (wpoint, bpoint, wperf, bperf)
async def game_update(self, game):
"""Called from Game.update_status()"""
if self.status == T_FINISHED and self.status != T_ARCHIVED:
return
wplayer = self.players[game.wplayer]
bplayer = self.players[game.bplayer]
if game.wberserk:
wplayer.nb_berserk += 1
self.nb_berserk += 1
if game.bberserk:
bplayer.nb_berserk += 1
self.nb_berserk += 1
if game.variant == "janggi":
wpoint, bpoint, wperf, bperf = self.points_perfs_janggi(game)
else:
wpoint, bpoint, wperf, bperf = self.points_perfs(game)
wplayer.points[-1] = wpoint
bplayer.points[-1] = bpoint
if wpoint[1] == STREAK and len(wplayer.points) >= 2:
wplayer.points[-2] = (wplayer.points[-2][0], STREAK)
if bpoint[1] == STREAK and len(bplayer.points) >= 2:
bplayer.points[-2] = (bplayer.points[-2][0], STREAK)
wplayer.rating = game.white_rating.rating_prov[0] + (int(game.wrdiff) if game.wrdiff else 0)
bplayer.rating = game.black_rating.rating_prov[0] + (int(game.brdiff) if game.brdiff else 0)
# TODO: in Swiss we will need Berger instead of performance to calculate tie breaks
nb = wplayer.nb_games
wplayer.performance = int(round((wplayer.performance * (nb - 1) + wperf) / nb, 0))
nb = bplayer.nb_games
bplayer.performance = int(round((bplayer.performance * (nb - 1) + bperf) / nb, 0))
wpscore = self.leaderboard.get(game.wplayer) // SCORE_SHIFT
self.leaderboard.update(
{game.wplayer: SCORE_SHIFT * (wpscore + wpoint[0]) + wplayer.performance}
)
bpscore = self.leaderboard.get(game.bplayer) // SCORE_SHIFT
self.leaderboard.update(
{game.bplayer: SCORE_SHIFT * (bpscore + bpoint[0]) + bplayer.performance}
)
self.nb_games_finished += 1
if game.result == "1-0":
self.w_win += 1
elif game.result == "0-1":
self.b_win += 1
elif game.result == "1/2-1/2":
self.draw += 1
asyncio.create_task(self.delayed_free(game, wplayer, bplayer))
# TODO: save player points to db
# await self.db_update_player(wplayer, self.players[wplayer])
# await self.db_update_player(bplayer, self.players[bplayer])
self.set_top_player()
await self.broadcast(
{
"type": "game_update",
"wname": game.wplayer.username,
"bname": game.bplayer.username,
}
)
if self.top_game is not None and self.top_game.id == game.id:
response = {
"type": "gameEnd",
"status": game.status,
"result": game.result,
"gameId": game.id,
}
await self.broadcast(response)
if (self.top_player is not None) and self.top_player.username not in (
game.wplayer.username,
game.bplayer.username,
):
top_game_candidate = self.players[self.top_player].games[-1]
if top_game_candidate.status != BYEGAME:
self.top_game = top_game_candidate
if (self.top_game is not None) and (self.top_game.status <= STARTED):
tgj = self.top_game_json
await self.broadcast(tgj)
async def delayed_free(self, game, wplayer, bplayer):
if self.system == ARENA:
await asyncio.sleep(3)
wplayer.free = True
bplayer.free = True
if game.status == FLAG:
# pause players when they don't start their game
if game.board.ply == 0:
wplayer.paused = True
elif game.board.ply == 1:
bplayer.paused = True
self.ongoing_games -= 1
async def broadcast(self, response):
for spectator in self.spectators:
try:
for ws in spectator.tournament_sockets[self.id]:
try:
await ws.send_json(response)
except ConnectionResetError:
pass
except KeyError:
# spectator was removed
pass
except Exception:
log.exception("Exception in tournament broadcast()")
async def db_update_player(self, user, player_data):
if self.app["db"] is None:
return
player_id = player_data.id
player_table = self.app["db"].tournament_player
if player_data.id is None: # new player join
player_id = await new_id(player_table)
player_data.id = player_id
if player_data.withdrawn:
new_data = {
"wd": True,
}
else:
full_score = self.leaderboard[user]
new_data = {
"_id": player_id,
"tid": self.id,
"uid": user.username,
"r": player_data.rating,
"pr": player_data.provisional,
"a": player_data.paused,
"f": player_data.win_streak == 2,
"s": int(full_score / SCORE_SHIFT),
"g": player_data.nb_games,
"w": player_data.nb_win,
"b": player_data.nb_berserk,
"e": player_data.performance,
"p": player_data.points,
"wd": False,
}
try:
print(
await player_table.find_one_and_update(
{"_id": player_id},
{"$set": new_data},
upsert=True,
return_document=ReturnDocument.AFTER,
)
)
except Exception:
if self.app["db"] is not None:
log.error(
"db find_one_and_update tournament_player %s into %s failed !!!",
player_id,
self.id,
)
new_data = {"nbPlayers": self.nb_players, "nbBerserk": self.nb_berserk}
print(
await self.app["db"].tournament.find_one_and_update(
{"_id": self.id},
{"$set": new_data},
return_document=ReturnDocument.AFTER,
)
)
async def save(self):
if self.app["db"] is None:
return
if self.nb_games_finished == 0:
print(await self.app["db"].tournament.delete_many({"_id": self.id}))
print("--- Deleted empty tournament %s" % self.id)
return
winner = self.leaderboard.peekitem(0)[0].username
new_data = {
"status": self.status,
"nbPlayers": self.nb_players,
"nbGames": self.nb_games_finished,
"winner": winner,
}
print(
await self.app["db"].tournament.find_one_and_update(
{"_id": self.id},
{"$set": new_data},
return_document=ReturnDocument.AFTER,
)
)
pairing_documents = []
pairing_table = self.app["db"].tournament_pairing
processed_games = set()
for user, user_data in self.players.items():
for game in user_data.games:
if game.status == BYEGAME: # ByeGame
continue
if game.id not in processed_games:
pairing_documents.append(
{
"_id": game.id,
"tid": self.id,
"u": (game.wplayer.username, game.bplayer.username),
"r": R2C[game.result],
"d": game.date,
"wr": game.wrating,
"br": game.brating,
"wb": game.wberserk,
"bb": game.bberserk,
}
)
processed_games.add(game.id)
await pairing_table.insert_many(pairing_documents)
for user in self.leaderboard:
await self.db_update_player(user, self.players[user])
if self.frequency == SHIELD:
variant_name = self.variant + ("960" if self.chess960 else "")
self.app["shield"][variant_name].append((winner, self.starts_at, self.id))
self.app["shield_owners"][variant_name] = winner
def print_leaderboard(self):
print("--- LEADERBOARD ---", self.id)
for player, full_score in self.leaderboard.items()[:10]:
print(
"%20s %4s %30s %2s %s"
% (
player.username,
self.players[player].rating,
self.players[player].points,
full_score,
self.players[player].performance,
)
)
@property
def create_discord_msg(self):
tc = time_control_str(self.base, self.inc, self.byoyomi_period)
tail960 = "960" if self.chess960 else ""
return "%s: **%s%s** %s tournament starts at UTC %s, duration will be **%s** minutes" % (
self.created_by,
self.variant,
tail960,
tc,
self.starts_at.strftime("%Y.%m.%d %H:%M"),
self.minutes,
)
def notify_discord_msg(self, minutes):
tc = time_control_str(self.base, self.inc, self.byoyomi_period)
tail960 = "960" if self.chess960 else ""
url = "https://www.pychess.org/tournament/%s" % self.id
if minutes >= 60:
time = int(minutes / 60)
time_text = "hours"
else:
time = minutes
time_text = "minutes"
return "**%s%s** %s tournament starts in **%s** %s! %s" % (
self.variant,
tail960,
tc,
time,
time_text,
url,
)
class SentenceRanker(object):
"""SentenceRanker manages a ranked list of sentences. Sentences are ranked based on
a heuristic. Currently there is one heuristic named "concept density", but this can
be expanded on in the future.
sent_id = (doc_id, position)
dict sentences_dict: Maps sent_id : ref to Sentence
dict concept_to_sentences: Maps a concept to the sentences in which it occurs
Concept : set([sent_ids])
ValueSortedDict ranks_to_sentences: [{sent_id : metric_value}] where list index corresponds to rank
dict concept_weights: original concept weights
k: Parameter that sets how many sentences are fed into ILP
"""
def __init__(self, sentences, concept_weights, summary_length, k, options):
'''
:param sentences: List of Sentence objects
:param concept_weights: Dictionary of weights for concept
:param k: Number of k sentences that is fed into the ILP per feedback iteration
'''
# Sets sentences_dict, concept_to_sent dict and ranked_to_sent dict
if options['strategy'] == STRATIFIED:
self.all_concept_weights = concept_weights
else:
self.all_concept_weights = deepcopy(
concept_weights) # keep original concept weights
self.initialize_sentences(sentences, concept_weights)
self.k = k
if options['relative_k']:
self.k = int(k * self.get_corpus_size())
self.k_is_dynamic = options['dynamic_k']
self.summary_length = summary_length
self.seen_sentences = set()
self.important_concepts = set()
# Interface for feedback
self.get_input_sentences = self.get_top_k_sentences
if options['strategy']:
self.init_strategy(options)
def initialize_sentences(self, sentences, concept_weights):
'''Initializes sentences based on metric "concept density".
creates:
sentences_dict, concept_to_sentences, ranks_to_sentences
'''
self.sentences_dict = {}
self.concept_to_sentences = defaultdict(set)
# Highest value --> first rank
self.ranks_to_sentences = ValueSortedDict(lambda x: -x)
for sent in sentences:
# Create sentences_dict
sent_id = (sent.doc_id, sent.position)
self.sentences_dict[sent_id] = sent
# Calculate concept density per sentence
concept_density = 0
for concept in sent.concepts:
concept_density += concept_weights[concept]
# Create concept2sent dic
self.concept_to_sentences[concept].add(sent_id)
concept_density /= float(sent.length)
# create ranks_to_sentences
self.ranks_to_sentences[sent_id] = concept_density
def update_ranking(self, new_accepts, new_rejects, new_implicits):
''' Changes top k sentences after feedback based on changed concept weights.'''
# TODO: implement implicits
changed_concepts = new_accepts + new_rejects
for concept in changed_concepts:
# Update affected sentences
for sent_id in self.concept_to_sentences[concept]:
sentence = self.sentences_dict[sent_id]
concept_density = 0
for c in sentence.concepts:
concept_density += self.all_concept_weights[c]
concept_density /= float(sentence.length)
# Update the metric and rank
self.ranks_to_sentences[sent_id] = concept_density
for concept in new_accepts:
self.important_concepts.add(concept)
if self.k_is_dynamic:
self.set_k()
self.k_history.append(self.k)
return
def update_weights(self, updated_weights):
'''Update weights that have changed after weights have been recalculated.'''
for key, value in updated_weights.items():
self.all_concept_weights[key] = value
return
def filter_concepts_of_top_k_sentences(self,
new_accepts=[],
new_rejects=[],
k=None,
sentences=None):
''' This method aggregates all relevant concepts based on top k sents and returns those.
The ILP should only receive those concepts that are also in the subset of sentences
which is passed to the ILP.
For the intermediate summary, which is generated for oracle types 'feedback_ilp',
and 'active_learning', the weights of the accepts and rejects should also be available
in the returned dictionary (they might not be part of top k sentences anymore).
'''
if sentences is None:
sentences = self.get_top_k_sentences(k)
concept_weights = {}
for sent in sentences:
for concept in sent.concepts:
concept_weights[concept] = self.all_concept_weights[concept]
if new_accepts + new_rejects:
for concept in new_accepts + new_rejects:
concept_weights[concept] = self.all_concept_weights[concept]
return concept_weights
def init_strategy(self, options):
if options['strategy'] == BY_TIME:
self.cost_model = CostModel('./algorithms/')
self.cost_model.k_to_constraints = self.k_to_constraint_size
self.determine_k = self.set_k_by_target_time
elif options['strategy'] == BY_ENTROPY_INIT:
self.k = self.set_k_by_entropy()
self.determine_k = lambda: self.k
elif options['strategy'] == BY_ENTROPY_ADAPT:
self.determine_k = self.set_k_by_entropy
elif options['strategy'] == BY_WINDOW:
self.adaptive_window_size = options['adaptive_window_size']
self.determine_k = self.get_important_sentences
elif options['strategy'] == BY_SWEEP:
self.sweep_threshold = options['sweep_threshold']
self.get_input_sentences = self.get_distinct_top_k_sentences
return
elif options['strategy'] == BY_POS_LINK:
self.get_input_sentences = self.get_sents_with_accepted_concepts
if options['dynamic_k']:
self.k_history = []
self.set_k()
self.k_history.append(self.k)
def set_k(self, k=None):
if k is None and self.k_is_dynamic:
chosen_k = self.determine_k()
# Set k so there are enough concepts to fill L
minimum_k = self.set_k_by_L()
self.k = max(chosen_k, minimum_k)
elif k is not None:
self.k = k
# Entropy Methods #
def get_entropy(self, sentences):
num_concepts = 0
concept_count = defaultdict(lambda: 0)
for sent in sentences:
for c in sent.concepts:
num_concepts += 1
concept_count[c] += 1
S = num_concepts # Size of Sample
self.num_of_concepts_in_summary = len(concept_count.keys())
base = len(self.all_concept_weights.keys())
entropy = 0.0
for concept, count in concept_count.items():
entropy -= (count / S) * log(
(count / S), base) * self.all_concept_weights[concept]
return entropy
def set_k_by_entropy(self):
k_to_entropy = []
top_k_sents = self.get_top_k_sentences(self.get_corpus_size())
num_concepts = 0
concept_count = defaultdict(lambda: 0)
base = len(self.all_concept_weights.keys())
for k, sent in enumerate(top_k_sents, 1):
for c in sent.concepts:
num_concepts += 1
concept_count[c] += 1
entropy = 0.0
S = num_concepts
for concept, count in concept_count.items():
entropy -= (count / S) * log(
(count / S), base) * self.all_concept_weights[concept]
k_to_entropy.append((k, entropy))
return max(k_to_entropy, key=lambda x: x[1])[0]
def set_k_by_L(self):
# TODO sequentially
for k in range(1, self.get_corpus_size() + 1):
bigram_count = len(
set([
c for sent in self.get_top_k_sentences(k)
for c in sent.concepts
]))
# We count unique bigrams, hence / 2
if bigram_count / 2 >= self.summary_length:
break
return k
def get_baseline_entropies(self):
# Max, min baselines
print(self.k_to_entropy)
return [
m(self.k_to_entropy, key=lambda x: x[1])[0] for m in [max, min]
]
def get_number_of_concepts(self):
return self.num_of_concepts_in_summary
# Time based strategy #
def set_k_by_target_time(self):
# TODO parametrize max_time
max_time = 1
target_constraint_size = int(self.time_to_ilp_constraints(max_time))
occurences = 0
concepts = set()
k = 0
while ((occurences + len(concepts) + 1) < target_constraint_size
or len(concepts) < self.summary_length / 2):
sent_id = self.ranks_to_sentences.iloc[k]
k += 1
for c in set(self.sentences_dict[sent_id].concepts):
occurences += 1
concepts.add(c)
return k
def k_to_constraint_size(self, k):
if type(k) is pd.core.series.Series:
s = {}
se = []
unique_k = set(k)
for i in unique_k:
# s.append(self.get_constraint_size_for(i))
s[i] = self.get_constraint_size_for(i)
for i in k:
se.append(s[i])
return pd.Series(se)
else:
return self.get_constraint_size_for(int(k))
def time_to_ilp_constraints(self, t):
return self.cost_model.constraints(t)
def get_constraint_size_for(self, k):
top_k_sent_ids = self.get_sentence_ids_for(k)
occurences = 0
concepts = set()
for s_id in top_k_sent_ids:
for ci in set(self.sentences_dict[s_id].concepts):
occurences += 1
concepts.add(ci)
return occurences + len(concepts) + 1
# Adaptive Window #
def get_important_sentences(self):
num_of_important_sents = 0
# Get number of consecutive sentences that have at least one important concept
for i, (sent_id,
metric_value) in enumerate(self.ranks_to_sentences.items()):
for concept in self.sentences_dict[sent_id].concepts:
if concept in self.important_concepts:
num_of_important_sents += 1
break
if (i + 1) != num_of_important_sents:
break
return int(num_of_important_sents * (1 + self.adaptive_window_size))
# Redundancy Sweep #
def get_distinct_top_k_sentences(self):
top_k_sent_ids = list(self.ranks_to_sentences.keys())
distinct_sentences = []
seen_concepts = defaultdict(lambda: False)
i = 0
while len(distinct_sentences) < self.k and i < self.get_corpus_size():
skip = False
sent = self.sentences_dict[top_k_sent_ids[i]]
counter = 0
for c in set(sent.concepts):
if seen_concepts[c]:
# +1 -> threshold = 0 means no overlap; 1 means one concept-overlap allowed
if counter >= self.sweep_threshold + 1:
skip = True
break
counter += 1
if not skip:
distinct_sentences.append(sent)
for c in sent.concepts:
seen_concepts[c] = 1
i += 1
return distinct_sentences
def get_sents_with_accepted_concepts(self):
input_sentences = self.get_top_k_sentences()
for sent_id in self.ranks_to_sentences.iloc[self.k:]:
sent = self.sentences_dict[sent_id]
for c in sent.concepts:
if c in self.important_concepts:
input_sentences.append(sent)
break
return input_sentences
def get_top_k_sentences(self, k=None):
print('### Top k %f' % self.k)
if k is None:
k = self.k
# Statistic purposes:
top_k_sent_ids = [
sent_id for sent_id in self.ranks_to_sentences.iloc[:k]
]
self.seen_sentences |= set(top_k_sent_ids)
return [
self.sentences_dict[sent_id]
for sent_id in self.ranks_to_sentences.iloc[:k]
]
def get_top_k_sentence_ids(self):
return self.ranks_to_sentences.iloc[:self.k]
def get_sentence_ids_for(self, k):
return self.ranks_to_sentences.iloc[:k]
def bisect_rank_by_value(self, value):
'''workaround method as ValueSortedDict bisects by key (not sensible for sort order by value).'''
self.ranks_to_sentences["bisect"] = value
index = self.ranks_to_sentences.index("bisect")
del self.ranks_to_sentences["bisect"]
return index
def get_corpus_size(self):
return len(self.sentences_dict)
# following: stuff for test purposes
def rank_to_metric(self, rank):
return self.ranks_to_sentences[self.ranks_to_sentences.iloc[rank]]
def print_ranked_sentences(self, n=10, full_sentence=False):
for rank, (doc_id,
position) in enumerate(self.get_top_k_sentence_ids()[:n],
1):
print("Rank: ", rank)
if full_sentence:
# self.sentences_dict[sent_id] returns the desired sentence
print(self.sentences_dict[(doc_id, position)].untokenized_form)
else:
print("doc_id: {}, sentence_pos: {}".format(doc_id, position))
# self.ranks_to_sentences[sent_id] returns the metric
print("Heuristic value: ",
self.ranks_to_sentences[(doc_id, position)])
print("#-----#")
def print_concept_map(self):
i = 0
for key, value in self.concept_to_sentences.items():
print(key, ":", self.concept_to_sentences[key])
for entry in self.concept_to_sentences[key]:
print(self.sentences_dict[entry].untokenized_form)
i += 1
print("-------")
if i >= 10:
break