def test_init_args():
temp = ValueSortedDict(enumerate(alphabet))
assert len(temp) == 26
assert temp[0] == 'a'
assert temp[25] == 'z'
assert temp.iloc[4] == 4
temp._check()
def initialize(self, parent=None):
# one model per cluster in current_state
self._models = {}
for cluster in parent.current_state.labels():
self._models[cluster] = self.compute_model(cluster, parent=parent)
# list of clusters
clusters = list(self._models)
try:
self._similarity = self.compute_similarity_matrix(parent=parent)
except NotImplementedError as e:
n_clusters = len(clusters)
self._similarity = ValueSortedDict()
for i, j in combinations(clusters, 2):
# compute similarity if (and only if) clusters are mergeable
if not parent.constraint.mergeable([i, j], parent=parent):
self._similarity[i, j] = -np.inf
self._similarity[j, i] = -np.inf
continue
similarity = self.compute_similarity(i, j, parent=parent)
self._similarity[i, j] = similarity
if not self.is_symmetric:
similarity = self.compute_similarity(j, i, parent=parent)
self._similarity[j, i] = similarity
def initialize_sentences(self, sentences, concept_weights):
'''Initializes sentences based on metric "concept density".
creates:
sentences_dict, concept_to_sentences, ranks_to_sentences
'''
self.sentences_dict = {}
self.concept_to_sentences = defaultdict(set)
# Highest value --> first rank
self.ranks_to_sentences = ValueSortedDict(lambda x: -x)
for sent in sentences:
# Create sentences_dict
sent_id = (sent.doc_id, sent.position)
self.sentences_dict[sent_id] = sent
# Calculate concept density per sentence
concept_density = 0
for concept in sent.concepts:
concept_density += concept_weights[concept]
# Create concept2sent dic
self.concept_to_sentences[concept].add(sent_id)
concept_density /= float(sent.length)
# create ranks_to_sentences
self.ranks_to_sentences[sent_id] = concept_density
async def startup(self, app):
self.app["highscore"] = {variant: ValueSortedDict(neg) for variant in VARIANTS}
self.app["highscore"]["crazyhouse960"] = ValueSortedDict(neg, ZH960)
self.wplayer = User(self.app, username="******", perfs=PERFS["user7"])
self.bplayer = User(self.app, username="******", perfs=PERFS["newplayer"])
self.strong_player = User(self.app, username="******", perfs=PERFS["strongplayer"])
self.weak_player = User(self.app, username="******", perfs=PERFS["weakplayer"])
def __init__(self, vertices: Iterable[any], compute_fills=True, compute_degrees=True):
"""
Create a Primal graph representing the interactions between vertices.
For min-fill or min-induced-width, use remove_and_process_node() instead of use remove_node()
:param vertices: The vertices of the graph
:param compute_fills; Whether this will be used to compute fills.
:param compute_degrees: Whether this will be used to compute degrees.
"""
self.connected_to: Dict[any, Set[any]] = {vertex: set() for vertex in vertices}
self._fills: ValueSortedDict[any, Tuple[int, List[Tuple[any, Set[any]]]]] = ValueSortedDict(lambda n: n[0]) if \
compute_fills else None
self._degrees: ValueSortedDict[any, int] = ValueSortedDict() if compute_degrees else None
def update_cpu_burn(name, count, t, l):
burn = burners.get(name, {})
burn['count'] = burn.get('count', 0) + count
burn['time'] = burn.get('time', 0.0) + t
if l is not None:
l = ValueSortedDict(l)
burn['list'] = burn.get('list', ValueSortedDict())
for k in l: # XXX replace this loop with .update()
burn['list'][k] = l[k]
length = len(burn['list'])
for _ in range(10, length):
burn['list'].popitem()
burners[name] = burn
async def create_new_pairings(self, waiting_players):
pairing = self.create_pairing(waiting_players)
if self.first_pairing:
self.first_pairing = False
# Before tournament starts leaderboard is ordered by ratings
# After first pairing it will be sorted by score points and performance
# so we have to make a clear (all 0) leaderboard here
new_leaderboard = [(user, 0) for user in self.leaderboard]
self.leaderboard = ValueSortedDict(neg, new_leaderboard)
self.leaderboard_keys_view = SortedKeysView(self.leaderboard)
games = await self.create_games(pairing)
return (pairing, games)
def load_scores(self) -> None:
"""
Loads all of the scores from the flairs on Reddit.
"""
self.scores = ValueSortedDict({})
for flair in self.bot.reddit.main_subreddit.flair(limit=None):
try:
self.scores[flair["user"].name.lower()] = int(
"".join([char for char in flair["flair_text"].split(" ")[0] if char.isnumeric()])
)
except Exception as e:
print(e)
pass
print("POINTS: Loaded scores.")
class FluidStreamDiscretizer():
def __init__(self, bin_count, history_length):
self.history = ValueSortedDict()
self.step = 0
self.thresholds = np.linspace(0, 1, bin_count - 1)
self.history_length = history_length
self.saturated = False
def __call__(self, value):
self.step += 1
self.history[self.step] = value
values = np.array(self.history.values())
bin_count = len(self.thresholds) + 1
self.thresholds = values[[
floor(idx * len(self.history) / bin_count)
for idx in range(1, bin_count)
]]
try:
del self.history[self.step - self.history_length]
self.saturated = True
except KeyError:
pass
return np.digitize(value, self.thresholds)
def __init__(self, palettes: List[OrderedSet], num_palettes: int,
colors_per_palette: int):
"""
:param palettes: A list of ordered sets of palettes
:param palette_relations: A list of lists of possible palettes for tiles
:param num_palettes: The number of palettes to reduce down to
:param colors_per_palette: The number of colors per palette
"""
self.was_run = False
self._palettes = [p.copy() for p in palettes]
self._current_number_of_palettes = len(palettes)
self._num_palettes = num_palettes
self._colors_per_palette = colors_per_palette
self._merges_performed: List[Tuple[int, int]] = []
self._count_per_pal = ValueSortedDict(
{pidx: -len(p)
for pidx, p in enumerate(self._palettes)})
def __init__(self, id, car, gui):
super(Field, self).__init__(target=self.dispatch, name='Field_{}_proc'.format(id))
self.exit = mp.Event()
self.id = id
self.car = car
self.gui = gui # type: FieldGUI
self.width = 150
self.height = 90
self.houses = dict()
self.xes = ValueSortedDict()
self.houseupdates = ValueSortedDict() # Essentially a priority queue with more functionality
self.update_idx = 0
self.next_new_house = 0
self.potentials = np.zeros((H, W), dtype = float)
self.que = mp.Queue(4)
self.path = list()
self.start()
def __init__(self, fname, restart=False):
if '/' not in fname:
fname = os.path.join(DIR, fname)
if not os.path.exists(os.path.dirname(fname)):
os.makedirs(os.path.dirname(fname))
self.fname = fname
print "Loading Recorder from", self.fname
self.record = ValueSortedDict(lambda x: -x[0])
self.write_freq = 10
self.read(restart)
self.write()
def identity_words(df):
# Analyze words, derive stats
good_counter = Counter()
good_count = 0
for batch in range(int(len(df) / 100000)):
sum_identities = np.sum(df.iloc[(batch) * 100000:(batch + 1) *
100000][REAL_IDENTITY_COLUMNS].values,
axis=1).astype('bool')
good_comments = ' '.join(df.iloc[(batch) * 100000:(batch + 1) *
100000][sum_identities == 0].apply(
lambda row: row['comment_text'],
axis=1)).split(' ')
good_counter.update(good_comments)
good_count += len(good_comments)
del good_comments
sum_identities = np.sum(df[REAL_IDENTITY_COLUMNS].values,
axis=1).astype('bool')
foul_comments = ' '.join(df[sum_identities > 0].apply(
lambda row: row['comment_text'], axis=1)).split(' ')
foul_counter = Counter(foul_comments)
foul_count = len(foul_comments)
del foul_comments
word_foulness = ValueSortedDict()
for word, count in good_counter.most_common() + foul_counter.most_common():
if count > MIN_FOUL_COUNT:
bad_freq = foul_counter[word] / foul_count
good_freq = (good_counter[word] + 1) / good_count
bad_prob = bad_freq / (good_freq)
word_foulness[word + '_'] = bad_prob
# Because of a weird bug we had to add an underscore to every word, let's remove it
foul_words = [
word[:-1] for word in list(
word_foulness.islice(word_foulness.bisect_key(FOUL_FREQ_TO_DROP)))
]
return foul_words
def knn_search(self, q=None, k=1):
self.q = q
self.visitedset = set()
self.candidates = ValueSortedDict()
self.result = ValueSortedDict()
count = 0
for i in range(self.m):
v_ep = self.corpus[np.random.choice(list(self.corpus.keys()))]
if self.dmat is None:
cost = self.switch_metric(self.q.values, v_ep.values)
else:
cost = self.dmat[q.index][v_ep.index]
count += 1
self.candidates[v_ep.index] = cost
self.visitedset.add(v_ep.index)
tempres = ValueSortedDict()
while True:
# get element c closest from candidates to q, and remove c
# from candidates
if len(self.candidates) > 0:
c = self.get_closest()
else:
break
# check stop condition
if len(self.result) >= k:
if self.check_stop_condition(c, k):
break
tempres.update(c)
# add neighbors of c if not in visitedset
c = self.corpus[list(c.keys())[0]]
for key in list(c.neighbors.keys()):
if key not in self.visitedset:
if self.dmat is None:
cost = self.switch_metric(self.q.values,
v_ep.values)
else:
cost = self.dmat[q.index][v_ep.index]
count += 1
self.visitedset.add(key)
self.candidates[key] = cost
tempres[key] = cost
# add tempres to result
self.result.update(tempres)
# return k neighbors/result
return self.result, count
def setUp(self):
self.loop = asyncio.get_event_loop()
self.app = {}
self.app["db"] = None
self.app["users"] = {}
self.app["games"] = {}
self.app["tasks"] = weakref.WeakSet()
self.app["crosstable"] = {}
self.app["highscore"] = {
variant: ValueSortedDict(neg)
for variant in VARIANTS
}
self.app["highscore"]["crazyhouse960"] = ValueSortedDict(neg, ZH960)
self.wplayer = User(self.app,
username="******",
perfs=PERFS["Doooovid"])
self.bplayer = User(self.app,
username="******",
perfs=PERFS["pepellou"])
self.splayer = User(self.app,
username="******",
perfs=PERFS["strongplayer"])
class Node:
def __init__(self, index: int, values: list, label=None):
self.index = index
self.values = values
self.label = label
self.neighbors = ValueSortedDict()
def __repr__(self):
return {'index': self.index, 'label': self.label}
def __str__(self):
return 'Node(index=' + str(self.index) + ', Label=' + str(
self.label) + ')'
def connect(self, index, cost, f):
"""
Calculate distance and store in a sorteddict
"""
# The dict would be sorted by values
self.neighbors[index] = cost
while len(self.neighbors) > f:
self.neighbors.popitem()
return self
def compute_similarity_matrix(self, parent=None):
clusters = list(self._models)
n_clusters = len(clusters)
X = np.vstack([self[cluster][0] for cluster in clusters])
nX = l2_normalize(X)
similarities = -squareform(pdist(nX, metric=self.distance))
matrix = ValueSortedDict()
for i, j in itertools.combinations(range(n_clusters), 2):
matrix[clusters[i], clusters[j]] = similarities[i, j]
matrix[clusters[j], clusters[i]] = similarities[j, i]
return matrix
def compute_similarities(self, cluster, clusters, parent=None):
x = self[cluster][0].reshape((1, -1))
X = np.vstack([self[c][0] for c in clusters])
# L2 normalization
nx = l2_normalize(x)
nX = l2_normalize(X)
similarities = -cdist(nx, nX, metric=self.distance)
matrix = ValueSortedDict()
for i, cluster_ in enumerate(clusters):
matrix[cluster, cluster_] = similarities[0, i]
matrix[cluster_, cluster] = similarities[0, i]
return matrix
def record_a_burn(name, start, url=None):
if isinstance(url, URL):
url = url.url
elapsed = time.process_time() - start
burn = burners.get(name, {})
burn['count'] = burn.get('count', 0) + 1
burn['time'] = burn.get('time', 0.0) + elapsed
avg = burn.get('avg', 10000000.)
# are we exceptional? 10x current average and significant
if elapsed > avg * 10 and elapsed > 0.015:
if 'list' not in burn:
burn['list'] = ValueSortedDict()
url = url or 'none'
burn['list'][url] = -elapsed
length = len(burn['list'])
for _ in range(10, length):
burn['list'].popitem()
burn['avg'] = burn['time'] / burn['count']
burners[name] = burn
def record_a_latency(name, start, url=None, elapsedmin=10.0):
if isinstance(url, URL):
url = url.url
elapsed = time.time() - start
latency = latencies.get(name, {})
latency['count'] = latency.get('count', 0) + 1
latency['time'] = latency.get('time', 0.0) + elapsed
if 'hist' not in latency:
latency['hist'] = HdrHistogram(1, 30 * 1000, 2) # 1ms-30sec, 2 sig figs
latency['hist'].record_value(elapsed * 1000) # ms
if elapsed > elapsedmin:
if 'list' not in latency:
latency['list'] = ValueSortedDict()
url = url or 'none'
length = len(latency['list'])
if length > 9:
for u in itertools.islice(latency['list'], 9, length):
del latency['list'][u]
latency['list'][url] = -elapsed
latencies[name] = latency
def compute_similarity_matrix(self, parent=None):
# name and number of clusters
clusters = list(self._models)
n_clusters = len(clusters)
# precompute pairwise embedding distance
data = parent.features
X = np.array(data[data.columns[2:]])
self.precomputed_ = -squareform(pdist(X, metric='euclidean'))
matrix = ValueSortedDict()
for i, j in itertools.combinations(range(n_clusters), 2):
# indices of embedding in ith cluster
indices_i = self[clusters[i]]
# indices of embedding in jth cluster
indices_j = self[clusters[j]]
# mean of all pairwise euclidean distances
similarity = np.mean(self.precomputed_[indices_i][:, indices_j])
matrix[clusters[i], clusters[j]] = similarity
matrix[clusters[j], clusters[i]] = similarity
return matrix
def record_a_latency(name, start, url=None, elapsedmin=10.0):
if isinstance(url, URL):
url = url.url
elapsed = time.time() - start
latency = latencies.get(name, {})
latency['count'] = latency.get('count', 0) + 1
latency['time'] = latency.get('time', 0.0) + elapsed
if 'hist' not in latency:
latency['hist'] = HdrHistogram(1, 30 * 1000,
2) # 1ms-30sec, 2 sig figs
latency['hist'].record_value(elapsed * 1000) # ms
# show the 10 most recent latencies > 10 seconds
if elapsed > elapsedmin:
if 'list' not in latency:
latency['list'] = ValueSortedDict()
url = url or 'none'
length = len(latency['list'])
for _ in range(9, length):
latency['list'].popitem(
last=False) # throwing away biggest value(s)
latency['list'][url] = -elapsed
latencies[name] = latency
def test_init():
temp = ValueSortedDict()
temp._check()
def boring_to_burners(d):
global burners
for k in d:
burners[k] = d[k]
burners[k]['list'] = ValueSortedDict(d[k].get('list', dict()))
def cluster_(self, fX):
"""Compute complete dendrogram
Parameters
----------
fX : (n_items, dimension) np.array
Embeddings.
Returns
-------
dendrogram : list of (i, j, distance) tuples
Dendrogram.
"""
N = len(fX)
# clusters contain the identifier of each cluster
clusters = SortedSet(np.arange(N))
# labels[i] = c means ith item belongs to cluster c
labels = np.array(np.arange(N))
squared = squareform(pdist(fX, metric=self.metric))
distances = ValueSortedDict()
for i, j in itertools.combinations(range(N), 2):
distances[i, j] = squared[i, j]
dendrogram = []
for _ in range(N-1):
# find most similar clusters
(c_i, c_j), d = distances.peekitem(index=0)
# keep track of this iteration
dendrogram.append((c_i, c_j, d))
# index of clusters in 'clusters' and 'fX'
i = clusters.index(c_i)
j = clusters.index(c_j)
# merge items of cluster c_j into cluster c_i
labels[labels == c_j] = c_i
# update c_i representative
fX[i] += fX[j]
# remove c_j cluster
fX[j:-1, :] = fX[j+1:, :]
fX = fX[:-1]
# remove distances to c_j cluster
for c in clusters[:j]:
distances.pop((c, c_j))
for c in clusters[j+1:]:
distances.pop((c_j, c))
clusters.remove(c_j)
if len(clusters) < 2:
continue
# compute distance to new c_i cluster
new_d = cdist(fX[i, :].reshape((1, -1)), fX, metric=self.metric).squeeze()
for c_k, d in zip(clusters, new_d):
if c_k < c_i:
distances[c_k, c_i] = d
elif c_k > c_i:
distances[c_i, c_k] = d
return dendrogram
async def init_state(app):
# We have to put "kill" into a dict to prevent getting:
# DeprecationWarning: Changing state of started or joined application is deprecated
app["data"] = {"kill": False}
if "db" not in app:
app["db"] = None
app["users"] = {
"Random-Mover": User(app, bot=True, username="******"),
"Fairy-Stockfish": User(app, bot=True, username="******"),
"Discord-Relay": User(app, anon=True, username="******"),
}
app["users"]["Random-Mover"].online = True
app["lobbysockets"] = {}
app["seeks"] = {}
app["games"] = {}
app["invites"] = {}
app["chat"] = collections.deque([], 100)
app["game_channels"] = set()
app["invite_channels"] = set()
app["highscore"] = {variant: ValueSortedDict(neg) for variant in VARIANTS}
app["crosstable"] = {}
app["stats"] = {}
# counters for games
app["g_cnt"] = 0
# last game played
app["tv"] = None
# fishnet active workers
app["workers"] = set()
# fishnet works
app["works"] = {}
# fishnet worker tasks
app["fishnet"] = asyncio.PriorityQueue()
# fishnet workers monitor
app["fishnet_monitor"] = {}
app["fishnet_versions"] = {}
for key in FISHNET_KEYS:
app["fishnet_monitor"][FISHNET_KEYS[key]] = collections.deque([], 50)
rm = app["users"]["Random-Mover"]
for variant in VARIANTS:
variant960 = variant.endswith("960")
variant_name = variant[:-3] if variant960 else variant
byoyomi = variant.endswith("shogi") or variant in ("dobutsu",
"gorogoro",
"janggi", "shogun")
seek = Seek(rm,
variant_name,
base=5,
inc=30 if byoyomi else 3,
level=0,
chess960=variant960,
byoyomi_period=1 if byoyomi else 0)
app["seeks"][seek.id] = seek
rm.seeks[seek.id] = seek
ai = app["users"]["Fairy-Stockfish"]
asyncio.create_task(BOT_task(ai, app))
asyncio.create_task(BOT_task(rm, app))
# Configure templating.
app["jinja"] = {}
base = os.path.dirname(__file__)
for lang in LANGUAGES:
# Generate compiled mo file
folder = os.path.join(base, "../lang/", lang, "LC_MESSAGES")
poname = os.path.join(folder, "server.po")
moname = os.path.join(folder, "server.mo")
try:
with open(poname, 'rb') as po_file:
po_lines = [
line for line in po_file if line[:8] != b"#, fuzzy"
]
mo = Msgfmt(po_lines).get()
with open(moname, 'wb') as mo_file:
mo_file.write(mo)
except PoSyntaxError:
log.error("PoSyntaxError in %s", poname)
# Create translation class
try:
translation = gettext.translation("server",
localedir="lang",
languages=[lang])
except FileNotFoundError:
log.warning("Missing translations file for lang %s", lang)
translation = gettext.NullTranslations()
env = jinja2.Environment(enable_async=True,
extensions=['jinja2.ext.i18n'],
loader=jinja2.FileSystemLoader("templates"),
autoescape=jinja2.select_autoescape(["html"]))
env.install_gettext_translations(translation, newstyle=True)
env.globals["static"] = static_url
app["jinja"][lang] = env
if app["db"] is None:
return
# Read users and highscore from db
try:
cursor = app["db"].user.find()
async for doc in cursor:
if doc["_id"] not in app["users"]:
perfs = doc.get("perfs")
if perfs is None:
perfs = {variant: DEFAULT_PERF for variant in VARIANTS}
app["users"][doc["_id"]] = User(
app,
username=doc["_id"],
title=doc.get("title"),
first_name=doc.get("first_name"),
last_name=doc.get("last_name"),
country=doc.get("country"),
bot=doc.get("title") == "BOT",
perfs=perfs,
enabled=doc.get("enabled", True))
db_collections = await app["db"].list_collection_names()
if "highscore" not in db_collections:
await generate_highscore(app["db"])
cursor = app["db"].highscore.find()
async for doc in cursor:
app["highscore"][doc["_id"]] = ValueSortedDict(neg, doc["scores"])
if "crosstable" not in db_collections:
await generate_crosstable(app["db"])
cursor = app["db"].crosstable.find()
async for doc in cursor:
app["crosstable"][doc["_id"]] = doc
await app["db"].game.create_index("us")
await app["db"].game.create_index("v")
await app["db"].game.create_index("y")
await app["db"].game.create_index("by")
except Exception:
print("Maybe mongodb is not running...")
raise
class Tournament(ABC):
"""Abstract base class for Arena/Swisss/RR Tournament classes
They have to implement create_pairing() for waiting_players"""
system: ClassVar[int] = ARENA
def __init__(
self,
app,
tournamentId,
variant="chess",
chess960=False,
rated=True,
before_start=5,
minutes=45,
name="",
description="",
fen="",
base=1,
inc=0,
byoyomi_period=0,
rounds=0,
created_by="",
created_at=None,
starts_at=None,
status=None,
with_clock=True,
frequency="",
):
self.app = app
self.id = tournamentId
self.name = name
self.description = description
self.variant = variant
self.rated = rated
self.before_start = before_start # in minutes
self.minutes = minutes # in minutes
self.fen = fen
self.base = base
self.inc = inc
self.byoyomi_period = byoyomi_period
self.chess960 = chess960
self.rounds = rounds
self.frequency = frequency
self.created_by = created_by
self.created_at = datetime.now(timezone.utc) if created_at is None else created_at
if starts_at == "" or starts_at is None:
self.starts_at = self.created_at + timedelta(seconds=int(before_start * 60))
else:
self.starts_at = starts_at
# TODO: calculate wave from TC, variant, number of players
self.wave = timedelta(seconds=3)
self.wave_delta = timedelta(seconds=1)
self.current_round = 0
self.prev_pairing = None
self.messages = collections.deque([], MAX_CHAT_LINES)
self.spectators = set()
self.players: dict[User, PlayerData] = {}
self.leaderboard = ValueSortedDict(neg)
self.leaderboard_keys_view = SortedKeysView(self.leaderboard)
self.status = T_CREATED if status is None else status
self.ongoing_games = 0
self.nb_players = 0
self.nb_games_finished = 0
self.w_win = 0
self.b_win = 0
self.draw = 0
self.nb_berserk = 0
self.nb_games_cached = -1
self.leaderboard_cache = {}
self.first_pairing = False
self.top_player = None
self.top_game = None
self.notify1 = False
self.notify2 = False
if minutes is None:
self.ends_at = self.starts_at + timedelta(days=1)
else:
self.ends_at = self.starts_at + timedelta(minutes=minutes)
if with_clock:
self.clock_task = asyncio.create_task(self.clock())
def __repr__(self):
return " ".join((self.id, self.name, self.created_at.isoformat()))
@abstractmethod
def create_pairing(self, waiting_players):
pass
def user_status(self, user):
if user in self.players:
return (
"paused"
if self.players[user].paused
else "withdrawn"
if self.players[user].withdrawn
else "joined"
)
else:
return "spectator"
def user_rating(self, user):
if user in self.players:
return self.players[user].rating
else:
return "%s%s" % user.get_rating(self.variant, self.chess960).rating_prov
def players_json(self, page=None, user=None):
if (page is None) and (user is not None) and (user in self.players):
if self.players[user].page > 0:
page = self.players[user].page
else:
div, mod = divmod(self.leaderboard.index(user) + 1, 10)
page = div + (1 if mod > 0 else 0)
if self.status == T_CREATED:
self.players[user].page = page
if page is None:
page = 1
if self.nb_games_cached != self.nb_games_finished:
# number of games changed (game ended)
self.leaderboard_cache = {}
self.nb_games_cached = self.nb_games_finished
elif user is not None:
if self.status == T_STARTED:
# player status changed (JOIN/PAUSE)
if page in self.leaderboard_cache:
del self.leaderboard_cache[page]
elif self.status == T_CREATED:
# number of players changed (JOIN/WITHDRAW)
self.leaderboard_cache = {}
if page in self.leaderboard_cache:
return self.leaderboard_cache[page]
def player_json(player, full_score):
return {
"paused": self.players[player].paused if self.status == T_STARTED else False,
"title": player.title,
"name": player.username,
"rating": self.players[player].rating,
"points": self.players[player].points,
"fire": self.players[player].win_streak,
"score": full_score, # SCORE_SHIFT-ed + performance rating
"perf": self.players[player].performance,
"nbGames": self.players[player].nb_games,
"nbWin": self.players[player].nb_win,
"nbBerserk": self.players[player].nb_berserk,
}
start = (page - 1) * 10
end = min(start + 10, self.nb_players)
page_json = {
"type": "get_players",
"requestedBy": user.username if user is not None else "",
"nbPlayers": self.nb_players,
"nbGames": self.nb_games_finished,
"page": page,
"players": [
player_json(player, full_score)
for player, full_score in self.leaderboard.items()[start:end]
],
}
if self.status > T_STARTED:
page_json["podium"] = [
player_json(player, full_score)
for player, full_score in self.leaderboard.items()[0:3]
]
self.leaderboard_cache[page] = page_json
return page_json
# TODO: cache this
def games_json(self, player_name):
player = self.app["users"].get(player_name)
return {
"type": "get_games",
"rank": self.leaderboard.index(player) + 1,
"title": player.title,
"name": player_name,
"perf": self.players[player].performance,
"nbGames": self.players[player].nb_games,
"nbWin": self.players[player].nb_win,
"nbBerserk": self.players[player].nb_berserk,
"games": [game.game_json(player) for game in self.players[player].games],
}
@property
def spectator_list(self):
return spectators(self)
@property
def top_game_json(self):
return {
"type": "top_game",
"gameId": self.top_game.id,
"variant": self.top_game.variant,
"fen": self.top_game.board.fen,
"w": self.top_game.wplayer.username,
"b": self.top_game.bplayer.username,
"wr": self.leaderboard_keys_view.index(self.top_game.wplayer) + 1,
"br": self.leaderboard_keys_view.index(self.top_game.bplayer) + 1,
"chess960": self.top_game.chess960,
"base": self.top_game.base,
"inc": self.top_game.inc,
"byoyomi": self.top_game.byoyomi_period,
}
def waiting_players(self):
return [
p
for p in self.leaderboard
if self.players[p].free
and self.id in p.tournament_sockets
and len(p.tournament_sockets[self.id]) > 0
and not self.players[p].paused
and not self.players[p].withdrawn
]
async def clock(self):
try:
while self.status not in (T_ABORTED, T_FINISHED, T_ARCHIVED):
now = datetime.now(timezone.utc)
if self.status == T_CREATED:
remaining_time = self.starts_at - now
remaining_mins_to_start = int(
((remaining_time.days * 3600 * 24) + remaining_time.seconds) / 60
)
if now >= self.starts_at:
if self.system != ARENA and len(self.players) < 3:
# Swiss and RR Tournaments need at least 3 players to start
await self.abort()
print("T_ABORTED: less than 3 player joined")
break
await self.start(now)
continue
elif (not self.notify2) and remaining_mins_to_start <= NOTIFY2_MINUTES:
self.notify1 = True
self.notify2 = True
await discord_message(
self.app,
"notify_tournament",
self.notify_discord_msg(remaining_mins_to_start),
)
continue
elif (not self.notify1) and remaining_mins_to_start <= NOTIFY1_MINUTES:
self.notify1 = True
await discord_message(
self.app,
"notify_tournament",
self.notify_discord_msg(remaining_mins_to_start),
)
continue
elif (self.minutes is not None) and now >= self.ends_at:
await self.finish()
print("T_FINISHED: no more time left")
break
elif self.status == T_STARTED:
if self.system == ARENA:
# In case of server restart
if self.prev_pairing is None:
self.prev_pairing = now - self.wave
if now >= self.prev_pairing + self.wave + random.uniform(
-self.wave_delta, self.wave_delta
):
waiting_players = self.waiting_players()
nb_waiting_players = len(waiting_players)
if nb_waiting_players >= 2:
log.debug("Enough player (%s), do pairing", nb_waiting_players)
await self.create_new_pairings(waiting_players)
self.prev_pairing = now
else:
log.debug(
"Too few player (%s) to make pairing",
nb_waiting_players,
)
else:
log.debug("Waiting for new pairing wave...")
elif self.ongoing_games == 0:
if self.current_round < self.rounds:
self.current_round += 1
log.debug("Do %s. round pairing", self.current_round)
waiting_players = self.waiting_players()
await self.create_new_pairings(waiting_players)
else:
await self.finish()
log.debug("T_FINISHED: no more round left")
break
else:
print(
"%s has %s ongoing game(s)..."
% (
"RR" if self.system == RR else "Swiss",
self.ongoing_games,
)
)
log.debug("%s CLOCK %s", self.id, now.strftime("%H:%M:%S"))
await asyncio.sleep(1)
except Exception:
log.exception("Exception in tournament clock()")
async def start(self, now):
self.status = T_STARTED
self.first_pairing = True
self.set_top_player()
response = {
"type": "tstatus",
"tstatus": self.status,
"secondsToFinish": (self.ends_at - now).total_seconds(),
}
await self.broadcast(response)
# force first pairing wave in arena
if self.system == ARENA:
self.prev_pairing = now - self.wave
if self.app["db"] is not None:
print(
await self.app["db"].tournament.find_one_and_update(
{"_id": self.id},
{"$set": {"status": self.status}},
return_document=ReturnDocument.AFTER,
)
)
@property
def summary(self):
return {
"type": "tstatus",
"tstatus": self.status,
"nbPlayers": self.nb_players,
"nbGames": self.nb_games_finished,
"wWin": self.w_win,
"bWin": self.b_win,
"draw": self.draw,
"berserk": self.nb_berserk,
"sumRating": sum(
self.players[player].rating
for player in self.players
if not self.players[player].withdrawn
),
}
async def finalize(self, status):
self.status = status
if len(self.players) > 0:
self.print_leaderboard()
print("--- TOURNAMENT RESULT ---")
for i in range(min(3, len(self.leaderboard))):
player = self.leaderboard.peekitem(i)[0]
print("--- #%s ---" % (i + 1), player.username)
# remove latest games from players tournament if it was not finished in time
for player in self.players:
if len(self.players[player].games) == 0:
continue
latest = self.players[player].games[-1]
if latest and latest.status in (CREATED, STARTED):
self.players[player].games.pop()
self.players[player].points.pop()
self.players[player].nb_games -= 1
# force to create new players json data
self.nb_games_cached = -1
await self.broadcast(self.summary)
await self.save()
await self.broadcast_spotlight()
async def broadcast_spotlight(self):
spotlights = tournament_spotlights(self.app["tournaments"])
lobby_sockets = self.app["lobbysockets"]
response = {"type": "spotlights", "items": spotlights}
await lobby_broadcast(lobby_sockets, response)
async def abort(self):
await self.finalize(T_ABORTED)
async def finish(self):
await self.finalize(T_FINISHED)
async def join(self, user):
if user.anon:
return
if self.system == RR and len(self.players) > self.rounds + 1:
raise EnoughPlayer
if user not in self.players:
# new player joined
rating, provisional = user.get_rating(self.variant, self.chess960).rating_prov
self.players[user] = PlayerData(rating, provisional)
elif self.players[user].withdrawn:
# withdrawn player joined again
rating, provisional = user.get_rating(self.variant, self.chess960).rating_prov
if user not in self.leaderboard:
# new player joined or withdrawn player joined again
if self.status == T_CREATED:
self.leaderboard.setdefault(user, rating)
else:
self.leaderboard.setdefault(user, 0)
self.nb_players += 1
self.players[user].paused = False
self.players[user].withdrawn = False
response = self.players_json(user=user)
await self.broadcast(response)
if self.status == T_CREATED:
await self.broadcast_spotlight()
await self.db_update_player(user, self.players[user])
async def withdraw(self, user):
self.players[user].withdrawn = True
self.leaderboard.pop(user)
self.nb_players -= 1
response = self.players_json(user=user)
await self.broadcast(response)
await self.broadcast_spotlight()
await self.db_update_player(user, self.players[user])
async def pause(self, user):
self.players[user].paused = True
# pause is different from withdraw and join because pause can be initiated from finished games page as well
response = self.players_json(user=user)
await self.broadcast(response)
if (self.top_player is not None) and self.top_player.username == user.username:
self.set_top_player()
await self.db_update_player(user, self.players[user])
def spactator_join(self, spectator):
self.spectators.add(spectator)
def spactator_leave(self, spectator):
self.spectators.discard(spectator)
async def create_new_pairings(self, waiting_players):
pairing = self.create_pairing(waiting_players)
if self.first_pairing:
self.first_pairing = False
# Before tournament starts leaderboard is ordered by ratings
# After first pairing it will be sorted by score points and performance
# so we have to make a clear (all 0) leaderboard here
new_leaderboard = [(user, 0) for user in self.leaderboard]
self.leaderboard = ValueSortedDict(neg, new_leaderboard)
self.leaderboard_keys_view = SortedKeysView(self.leaderboard)
games = await self.create_games(pairing)
return (pairing, games)
def set_top_player(self):
idx = 0
self.top_player = None
while idx < self.nb_players:
top_player = self.leaderboard.peekitem(idx)[0]
if self.players[top_player].paused:
idx += 1
continue
else:
self.top_player = top_player
break
async def create_games(self, pairing):
check_top_game = self.top_player is not None
new_top_game = False
games = []
game_table = None if self.app["db"] is None else self.app["db"].game
for wp, bp in pairing:
game_id = await new_id(game_table)
game = Game(
self.app,
game_id,
self.variant,
self.fen,
wp,
bp,
base=self.base,
inc=self.inc,
byoyomi_period=self.byoyomi_period,
rated=RATED if self.rated else CASUAL,
tournamentId=self.id,
chess960=self.chess960,
)
games.append(game)
self.app["games"][game_id] = game
await insert_game_to_db(game, self.app)
# TODO: save new game to db
if 0: # self.app["db"] is not None:
doc = {
"_id": game.id,
"tid": self.id,
"u": [game.wplayer.username, game.bplayer.username],
"r": "*",
"d": game.date,
"wr": game.wrating,
"br": game.brating,
}
await self.app["db"].tournament_pairing.insert_one(doc)
self.players[wp].games.append(game)
self.players[bp].games.append(game)
self.players[wp].points.append("*")
self.players[bp].points.append("*")
self.ongoing_games += 1
self.players[wp].free = False
self.players[bp].free = False
self.players[wp].nb_games += 1
self.players[bp].nb_games += 1
self.players[wp].prev_opp = game.bplayer.username
self.players[bp].prev_opp = game.wplayer.username
self.players[wp].color_balance += 1
self.players[bp].color_balance -= 1
self.players[wp].nb_not_paired = 0
self.players[bp].nb_not_paired = 0
response = {
"type": "new_game",
"gameId": game_id,
"wplayer": wp.username,
"bplayer": bp.username,
}
try:
ws = next(iter(wp.tournament_sockets[self.id]))
if ws is not None:
await ws.send_json(response)
except Exception:
self.pause(wp)
log.debug("White player %s left the tournament", wp.username)
try:
ws = next(iter(bp.tournament_sockets[self.id]))
if ws is not None:
await ws.send_json(response)
except Exception:
self.pause(bp)
log.debug("Black player %s left the tournament", bp.username)
if (
check_top_game
and (self.top_player is not None)
and self.top_player.username in (game.wplayer.username, game.bplayer.username)
and game.status != BYEGAME
): # Bye game
self.top_game = game
check_top_game = False
new_top_game = True
if new_top_game:
tgj = self.top_game_json
await self.broadcast(tgj)
return games
def points_perfs(self, game: Game) -> Tuple[Point, Point, int, int]:
wplayer = self.players[game.wplayer]
bplayer = self.players[game.bplayer]
wpoint = (0, SCORE)
bpoint = (0, SCORE)
wperf = game.black_rating.rating_prov[0]
bperf = game.white_rating.rating_prov[0]
if game.result == "1/2-1/2":
if self.system == ARENA:
if game.board.ply > 10:
wpoint = (2, SCORE) if wplayer.win_streak == 2 else (1, SCORE)
bpoint = (2, SCORE) if bplayer.win_streak == 2 else (1, SCORE)
wplayer.win_streak = 0
bplayer.win_streak = 0
else:
wpoint, bpoint = (1, SCORE), (1, SCORE)
elif game.result == "1-0":
wplayer.nb_win += 1
if self.system == ARENA:
if wplayer.win_streak == 2:
wpoint = (4, DOUBLE)
else:
wplayer.win_streak += 1
wpoint = (2, STREAK if wplayer.win_streak == 2 else SCORE)
bplayer.win_streak = 0
else:
wpoint = (2, SCORE)
if game.wberserk and game.board.ply >= 13:
wpoint = (wpoint[0] + 1, wpoint[1])
wperf += 500
bperf -= 500
elif game.result == "0-1":
bplayer.nb_win += 1
if self.system == ARENA:
if bplayer.win_streak == 2:
bpoint = (4, DOUBLE)
else:
bplayer.win_streak += 1
bpoint = (2, STREAK if bplayer.win_streak == 2 else SCORE)
wplayer.win_streak = 0
else:
bpoint = (2, SCORE)
if game.bberserk and game.board.ply >= 14:
bpoint = (bpoint[0] + 1, bpoint[1])
wperf -= 500
bperf += 500
return (wpoint, bpoint, wperf, bperf)
def points_perfs_janggi(self, game):
wplayer = self.players[game.wplayer]
bplayer = self.players[game.bplayer]
wpoint = (0, SCORE)
bpoint = (0, SCORE)
wperf = game.black_rating.rating_prov[0]
bperf = game.white_rating.rating_prov[0]
if game.status == VARIANTEND:
wplayer.win_streak = 0
bplayer.win_streak = 0
if game.result == "1-0":
if self.system == ARENA:
wpoint = (4 * 2 if wplayer.win_streak == 2 else 4, SCORE)
bpoint = (2 * 2 if bplayer.win_streak == 2 else 2, SCORE)
else:
wpoint = (4, SCORE)
bpoint = (2, SCORE)
elif game.result == "0-1":
if self.system == ARENA:
bpoint = (4 * 2 if bplayer.win_streak == 2 else 4, SCORE)
wpoint = (2 * 2 if wplayer.win_streak == 2 else 2, SCORE)
else:
bpoint = (4, SCORE)
wpoint = (2, SCORE)
elif game.result == "1-0":
wplayer.nb_win += 1
if self.system == ARENA:
if wplayer.win_streak == 2:
wpoint = (7 * 2, DOUBLE)
else:
wplayer.win_streak += 1
wpoint = (7, STREAK if wplayer.win_streak == 2 else SCORE)
bplayer.win_streak = 0
if game.wberserk and game.board.ply >= 13:
wpoint = (wpoint[0] + 3, wpoint[1])
else:
wpoint = (7, SCORE)
bpoint = (0, SCORE)
wperf += 500
bperf -= 500
elif game.result == "0-1":
bplayer.nb_win += 1
if self.system == ARENA:
if bplayer.win_streak == 2:
bpoint = (7 * 2, DOUBLE)
else:
bplayer.win_streak += 1
bpoint = (7, STREAK if bplayer.win_streak == 2 else SCORE)
wplayer.win_streak = 0
if game.bberserk and game.board.ply >= 14:
bpoint = (bpoint[0] + 3, bpoint[1])
else:
wpoint = (0, SCORE)
bpoint = (7, SCORE)
wperf -= 500
bperf += 500
return (wpoint, bpoint, wperf, bperf)
async def game_update(self, game):
"""Called from Game.update_status()"""
if self.status == T_FINISHED and self.status != T_ARCHIVED:
return
wplayer = self.players[game.wplayer]
bplayer = self.players[game.bplayer]
if game.wberserk:
wplayer.nb_berserk += 1
self.nb_berserk += 1
if game.bberserk:
bplayer.nb_berserk += 1
self.nb_berserk += 1
if game.variant == "janggi":
wpoint, bpoint, wperf, bperf = self.points_perfs_janggi(game)
else:
wpoint, bpoint, wperf, bperf = self.points_perfs(game)
wplayer.points[-1] = wpoint
bplayer.points[-1] = bpoint
if wpoint[1] == STREAK and len(wplayer.points) >= 2:
wplayer.points[-2] = (wplayer.points[-2][0], STREAK)
if bpoint[1] == STREAK and len(bplayer.points) >= 2:
bplayer.points[-2] = (bplayer.points[-2][0], STREAK)
wplayer.rating = game.white_rating.rating_prov[0] + (int(game.wrdiff) if game.wrdiff else 0)
bplayer.rating = game.black_rating.rating_prov[0] + (int(game.brdiff) if game.brdiff else 0)
# TODO: in Swiss we will need Berger instead of performance to calculate tie breaks
nb = wplayer.nb_games
wplayer.performance = int(round((wplayer.performance * (nb - 1) + wperf) / nb, 0))
nb = bplayer.nb_games
bplayer.performance = int(round((bplayer.performance * (nb - 1) + bperf) / nb, 0))
wpscore = self.leaderboard.get(game.wplayer) // SCORE_SHIFT
self.leaderboard.update(
{game.wplayer: SCORE_SHIFT * (wpscore + wpoint[0]) + wplayer.performance}
)
bpscore = self.leaderboard.get(game.bplayer) // SCORE_SHIFT
self.leaderboard.update(
{game.bplayer: SCORE_SHIFT * (bpscore + bpoint[0]) + bplayer.performance}
)
self.nb_games_finished += 1
if game.result == "1-0":
self.w_win += 1
elif game.result == "0-1":
self.b_win += 1
elif game.result == "1/2-1/2":
self.draw += 1
asyncio.create_task(self.delayed_free(game, wplayer, bplayer))
# TODO: save player points to db
# await self.db_update_player(wplayer, self.players[wplayer])
# await self.db_update_player(bplayer, self.players[bplayer])
self.set_top_player()
await self.broadcast(
{
"type": "game_update",
"wname": game.wplayer.username,
"bname": game.bplayer.username,
}
)
if self.top_game is not None and self.top_game.id == game.id:
response = {
"type": "gameEnd",
"status": game.status,
"result": game.result,
"gameId": game.id,
}
await self.broadcast(response)
if (self.top_player is not None) and self.top_player.username not in (
game.wplayer.username,
game.bplayer.username,
):
top_game_candidate = self.players[self.top_player].games[-1]
if top_game_candidate.status != BYEGAME:
self.top_game = top_game_candidate
if (self.top_game is not None) and (self.top_game.status <= STARTED):
tgj = self.top_game_json
await self.broadcast(tgj)
async def delayed_free(self, game, wplayer, bplayer):
if self.system == ARENA:
await asyncio.sleep(3)
wplayer.free = True
bplayer.free = True
if game.status == FLAG:
# pause players when they don't start their game
if game.board.ply == 0:
wplayer.paused = True
elif game.board.ply == 1:
bplayer.paused = True
self.ongoing_games -= 1
async def broadcast(self, response):
for spectator in self.spectators:
try:
for ws in spectator.tournament_sockets[self.id]:
try:
await ws.send_json(response)
except ConnectionResetError:
pass
except KeyError:
# spectator was removed
pass
except Exception:
log.exception("Exception in tournament broadcast()")
async def db_update_player(self, user, player_data):
if self.app["db"] is None:
return
player_id = player_data.id
player_table = self.app["db"].tournament_player
if player_data.id is None: # new player join
player_id = await new_id(player_table)
player_data.id = player_id
if player_data.withdrawn:
new_data = {
"wd": True,
}
else:
full_score = self.leaderboard[user]
new_data = {
"_id": player_id,
"tid": self.id,
"uid": user.username,
"r": player_data.rating,
"pr": player_data.provisional,
"a": player_data.paused,
"f": player_data.win_streak == 2,
"s": int(full_score / SCORE_SHIFT),
"g": player_data.nb_games,
"w": player_data.nb_win,
"b": player_data.nb_berserk,
"e": player_data.performance,
"p": player_data.points,
"wd": False,
}
try:
print(
await player_table.find_one_and_update(
{"_id": player_id},
{"$set": new_data},
upsert=True,
return_document=ReturnDocument.AFTER,
)
)
except Exception:
if self.app["db"] is not None:
log.error(
"db find_one_and_update tournament_player %s into %s failed !!!",
player_id,
self.id,
)
new_data = {"nbPlayers": self.nb_players, "nbBerserk": self.nb_berserk}
print(
await self.app["db"].tournament.find_one_and_update(
{"_id": self.id},
{"$set": new_data},
return_document=ReturnDocument.AFTER,
)
)
async def save(self):
if self.app["db"] is None:
return
if self.nb_games_finished == 0:
print(await self.app["db"].tournament.delete_many({"_id": self.id}))
print("--- Deleted empty tournament %s" % self.id)
return
winner = self.leaderboard.peekitem(0)[0].username
new_data = {
"status": self.status,
"nbPlayers": self.nb_players,
"nbGames": self.nb_games_finished,
"winner": winner,
}
print(
await self.app["db"].tournament.find_one_and_update(
{"_id": self.id},
{"$set": new_data},
return_document=ReturnDocument.AFTER,
)
)
pairing_documents = []
pairing_table = self.app["db"].tournament_pairing
processed_games = set()
for user, user_data in self.players.items():
for game in user_data.games:
if game.status == BYEGAME: # ByeGame
continue
if game.id not in processed_games:
pairing_documents.append(
{
"_id": game.id,
"tid": self.id,
"u": (game.wplayer.username, game.bplayer.username),
"r": R2C[game.result],
"d": game.date,
"wr": game.wrating,
"br": game.brating,
"wb": game.wberserk,
"bb": game.bberserk,
}
)
processed_games.add(game.id)
await pairing_table.insert_many(pairing_documents)
for user in self.leaderboard:
await self.db_update_player(user, self.players[user])
if self.frequency == SHIELD:
variant_name = self.variant + ("960" if self.chess960 else "")
self.app["shield"][variant_name].append((winner, self.starts_at, self.id))
self.app["shield_owners"][variant_name] = winner
def print_leaderboard(self):
print("--- LEADERBOARD ---", self.id)
for player, full_score in self.leaderboard.items()[:10]:
print(
"%20s %4s %30s %2s %s"
% (
player.username,
self.players[player].rating,
self.players[player].points,
full_score,
self.players[player].performance,
)
)
@property
def create_discord_msg(self):
tc = time_control_str(self.base, self.inc, self.byoyomi_period)
tail960 = "960" if self.chess960 else ""
return "%s: **%s%s** %s tournament starts at UTC %s, duration will be **%s** minutes" % (
self.created_by,
self.variant,
tail960,
tc,
self.starts_at.strftime("%Y.%m.%d %H:%M"),
self.minutes,
)
def notify_discord_msg(self, minutes):
tc = time_control_str(self.base, self.inc, self.byoyomi_period)
tail960 = "960" if self.chess960 else ""
url = "https://www.pychess.org/tournament/%s" % self.id
if minutes >= 60:
time = int(minutes / 60)
time_text = "hours"
else:
time = minutes
time_text = "minutes"
return "**%s%s** %s tournament starts in **%s** %s! %s" % (
self.variant,
tail960,
tc,
time,
time_text,
url,
)
def __init__(
self,
app,
tournamentId,
variant="chess",
chess960=False,
rated=True,
before_start=5,
minutes=45,
name="",
description="",
fen="",
base=1,
inc=0,
byoyomi_period=0,
rounds=0,
created_by="",
created_at=None,
starts_at=None,
status=None,
with_clock=True,
frequency="",
):
self.app = app
self.id = tournamentId
self.name = name
self.description = description
self.variant = variant
self.rated = rated
self.before_start = before_start # in minutes
self.minutes = minutes # in minutes
self.fen = fen
self.base = base
self.inc = inc
self.byoyomi_period = byoyomi_period
self.chess960 = chess960
self.rounds = rounds
self.frequency = frequency
self.created_by = created_by
self.created_at = datetime.now(timezone.utc) if created_at is None else created_at
if starts_at == "" or starts_at is None:
self.starts_at = self.created_at + timedelta(seconds=int(before_start * 60))
else:
self.starts_at = starts_at
# TODO: calculate wave from TC, variant, number of players
self.wave = timedelta(seconds=3)
self.wave_delta = timedelta(seconds=1)
self.current_round = 0
self.prev_pairing = None
self.messages = collections.deque([], MAX_CHAT_LINES)
self.spectators = set()
self.players: dict[User, PlayerData] = {}
self.leaderboard = ValueSortedDict(neg)
self.leaderboard_keys_view = SortedKeysView(self.leaderboard)
self.status = T_CREATED if status is None else status
self.ongoing_games = 0
self.nb_players = 0
self.nb_games_finished = 0
self.w_win = 0
self.b_win = 0
self.draw = 0
self.nb_berserk = 0
self.nb_games_cached = -1
self.leaderboard_cache = {}
self.first_pairing = False
self.top_player = None
self.top_game = None
self.notify1 = False
self.notify2 = False
if minutes is None:
self.ends_at = self.starts_at + timedelta(days=1)
else:
self.ends_at = self.starts_at + timedelta(minutes=minutes)
if with_clock:
self.clock_task = asyncio.create_task(self.clock())
async def init_state(app):
# We have to put "kill" into a dict to prevent getting:
# DeprecationWarning: Changing state of started or joined application is deprecated
app["data"] = {"kill": False}
if "db" not in app:
app["db"] = None
app["users"] = {
"Random-Mover": User(app, bot=True, username="******"),
"Fairy-Stockfish": User(app, bot=True, username="******"),
"Discord-Relay": User(app, anon=True, username="******"),
}
app["users"]["Random-Mover"].bot_online = True
app["websockets"] = {}
app["seeks"] = {}
app["games"] = {}
app["chat"] = collections.deque([], 200)
app["channels"] = set()
app["highscore"] = {variant: ValueSortedDict(neg) for variant in VARIANTS}
app["crosstable"] = {}
# counters for games and users
app["g_cnt"] = 0
app["u_cnt"] = 0
# last game played
app["tv"] = None
# fishnet active workers
app["workers"] = set()
# fishnet works
app["works"] = {}
# fishnet worker tasks
app["fishnet"] = asyncio.PriorityQueue()
# fishnet workers monitor
app["fishnet_monitor"] = {}
app["fishnet_versions"] = {}
for key in FISHNET_KEYS:
app["fishnet_monitor"][FISHNET_KEYS[key]] = collections.deque([], 50)
rm = app["users"]["Random-Mover"]
for variant in VARIANTS:
variant960 = variant.endswith("960")
variant_name = variant[:-3] if variant960 else variant
byoyomi = variant == "janggi" or variant.endswith(
"shogi") or variant == "shogun"
seek = Seek(rm,
variant_name,
base=5,
inc=3,
level=0,
chess960=variant960,
byoyomi_period=1 if byoyomi else 0)
app["seeks"][seek.id] = seek
rm.seeks[seek.id] = seek
ai = app["users"]["Fairy-Stockfish"]
loop = asyncio.get_event_loop()
loop.create_task(BOT_task(ai, app))
loop.create_task(BOT_task(rm, app))
# Configure templating.
app["jinja"] = jinja2.Environment(
loader=jinja2.FileSystemLoader("templates"),
autoescape=jinja2.select_autoescape(["html"]))
if app["db"] is None:
return
# Read users and highscore from db
try:
cursor = app["db"].user.find()
async for doc in cursor:
if doc["_id"] not in app["users"]:
perfs = doc.get("perfs")
if perfs is None:
perfs = {variant: DEFAULT_PERF for variant in VARIANTS}
app["users"][doc["_id"]] = User(
app,
username=doc["_id"],
title=doc.get("title"),
first_name=doc.get("first_name"),
last_name=doc.get("last_name"),
country=doc.get("country"),
bot=doc.get("title") == "BOT",
perfs=perfs,
enabled=doc.get("enabled", True))
db_collections = await app["db"].list_collection_names()
if "highscore" not in db_collections:
await generate_highscore(app["db"])
cursor = app["db"].highscore.find()
async for doc in cursor:
app["highscore"][doc["_id"]] = ValueSortedDict(neg, doc["scores"])
if "crosstable" not in db_collections:
await generate_crosstable(app["db"])
cursor = app["db"].crosstable.find()
async for doc in cursor:
app["crosstable"][doc["_id"]] = doc
except Exception:
print("Maybe mongodb is not running...")
raise
def __init__(self, bin_count, history_length):
self.history = ValueSortedDict()
self.step = 0
self.thresholds = np.linspace(0, 1, bin_count - 1)
self.history_length = history_length
self.saturated = False
def test_copy():
temp = ValueSortedDict(identity, enumerate(reversed(alphabet)))
that = temp.copy()
assert temp == that
assert temp._key != that._key
def test_init_kwargs():
temp = ValueSortedDict(None, a=0, b=1, c=2)
assert len(temp) == 3
assert temp['a'] == 0
assert temp.iloc[0] == 'a'
temp._check()
def pull_foreign_keys(self, dim: DimensionName) -> None:
select_statement = self.foreign_key_lookups[dim] # type: Select
from db import fetch
self._foreign_keys[dim] = ValueSortedDict(
{row[0]: str(row[1])
for row in fetch(select_statement)})
class SentenceRanker(object):
"""SentenceRanker manages a ranked list of sentences. Sentences are ranked based on
a heuristic. Currently there is one heuristic named "concept density", but this can
be expanded on in the future.
sent_id = (doc_id, position)
dict sentences_dict: Maps sent_id : ref to Sentence
dict concept_to_sentences: Maps a concept to the sentences in which it occurs
Concept : set([sent_ids])
ValueSortedDict ranks_to_sentences: [{sent_id : metric_value}] where list index corresponds to rank
dict concept_weights: original concept weights
k: Parameter that sets how many sentences are fed into ILP
"""
def __init__(self, sentences, concept_weights, summary_length, k, options):
'''
:param sentences: List of Sentence objects
:param concept_weights: Dictionary of weights for concept
:param k: Number of k sentences that is fed into the ILP per feedback iteration
'''
# Sets sentences_dict, concept_to_sent dict and ranked_to_sent dict
if options['strategy'] == STRATIFIED:
self.all_concept_weights = concept_weights
else:
self.all_concept_weights = deepcopy(
concept_weights) # keep original concept weights
self.initialize_sentences(sentences, concept_weights)
self.k = k
if options['relative_k']:
self.k = int(k * self.get_corpus_size())
self.k_is_dynamic = options['dynamic_k']
self.summary_length = summary_length
self.seen_sentences = set()
self.important_concepts = set()
# Interface for feedback
self.get_input_sentences = self.get_top_k_sentences
if options['strategy']:
self.init_strategy(options)
def initialize_sentences(self, sentences, concept_weights):
'''Initializes sentences based on metric "concept density".
creates:
sentences_dict, concept_to_sentences, ranks_to_sentences
'''
self.sentences_dict = {}
self.concept_to_sentences = defaultdict(set)
# Highest value --> first rank
self.ranks_to_sentences = ValueSortedDict(lambda x: -x)
for sent in sentences:
# Create sentences_dict
sent_id = (sent.doc_id, sent.position)
self.sentences_dict[sent_id] = sent
# Calculate concept density per sentence
concept_density = 0
for concept in sent.concepts:
concept_density += concept_weights[concept]
# Create concept2sent dic
self.concept_to_sentences[concept].add(sent_id)
concept_density /= float(sent.length)
# create ranks_to_sentences
self.ranks_to_sentences[sent_id] = concept_density
def update_ranking(self, new_accepts, new_rejects, new_implicits):
''' Changes top k sentences after feedback based on changed concept weights.'''
# TODO: implement implicits
changed_concepts = new_accepts + new_rejects
for concept in changed_concepts:
# Update affected sentences
for sent_id in self.concept_to_sentences[concept]:
sentence = self.sentences_dict[sent_id]
concept_density = 0
for c in sentence.concepts:
concept_density += self.all_concept_weights[c]
concept_density /= float(sentence.length)
# Update the metric and rank
self.ranks_to_sentences[sent_id] = concept_density
for concept in new_accepts:
self.important_concepts.add(concept)
if self.k_is_dynamic:
self.set_k()
self.k_history.append(self.k)
return
def update_weights(self, updated_weights):
'''Update weights that have changed after weights have been recalculated.'''
for key, value in updated_weights.items():
self.all_concept_weights[key] = value
return
def filter_concepts_of_top_k_sentences(self,
new_accepts=[],
new_rejects=[],
k=None,
sentences=None):
''' This method aggregates all relevant concepts based on top k sents and returns those.
The ILP should only receive those concepts that are also in the subset of sentences
which is passed to the ILP.
For the intermediate summary, which is generated for oracle types 'feedback_ilp',
and 'active_learning', the weights of the accepts and rejects should also be available
in the returned dictionary (they might not be part of top k sentences anymore).
'''
if sentences is None:
sentences = self.get_top_k_sentences(k)
concept_weights = {}
for sent in sentences:
for concept in sent.concepts:
concept_weights[concept] = self.all_concept_weights[concept]
if new_accepts + new_rejects:
for concept in new_accepts + new_rejects:
concept_weights[concept] = self.all_concept_weights[concept]
return concept_weights
def init_strategy(self, options):
if options['strategy'] == BY_TIME:
self.cost_model = CostModel('./algorithms/')
self.cost_model.k_to_constraints = self.k_to_constraint_size
self.determine_k = self.set_k_by_target_time
elif options['strategy'] == BY_ENTROPY_INIT:
self.k = self.set_k_by_entropy()
self.determine_k = lambda: self.k
elif options['strategy'] == BY_ENTROPY_ADAPT:
self.determine_k = self.set_k_by_entropy
elif options['strategy'] == BY_WINDOW:
self.adaptive_window_size = options['adaptive_window_size']
self.determine_k = self.get_important_sentences
elif options['strategy'] == BY_SWEEP:
self.sweep_threshold = options['sweep_threshold']
self.get_input_sentences = self.get_distinct_top_k_sentences
return
elif options['strategy'] == BY_POS_LINK:
self.get_input_sentences = self.get_sents_with_accepted_concepts
if options['dynamic_k']:
self.k_history = []
self.set_k()
self.k_history.append(self.k)
def set_k(self, k=None):
if k is None and self.k_is_dynamic:
chosen_k = self.determine_k()
# Set k so there are enough concepts to fill L
minimum_k = self.set_k_by_L()
self.k = max(chosen_k, minimum_k)
elif k is not None:
self.k = k
# Entropy Methods #
def get_entropy(self, sentences):
num_concepts = 0
concept_count = defaultdict(lambda: 0)
for sent in sentences:
for c in sent.concepts:
num_concepts += 1
concept_count[c] += 1
S = num_concepts # Size of Sample
self.num_of_concepts_in_summary = len(concept_count.keys())
base = len(self.all_concept_weights.keys())
entropy = 0.0
for concept, count in concept_count.items():
entropy -= (count / S) * log(
(count / S), base) * self.all_concept_weights[concept]
return entropy
def set_k_by_entropy(self):
k_to_entropy = []
top_k_sents = self.get_top_k_sentences(self.get_corpus_size())
num_concepts = 0
concept_count = defaultdict(lambda: 0)
base = len(self.all_concept_weights.keys())
for k, sent in enumerate(top_k_sents, 1):
for c in sent.concepts:
num_concepts += 1
concept_count[c] += 1
entropy = 0.0
S = num_concepts
for concept, count in concept_count.items():
entropy -= (count / S) * log(
(count / S), base) * self.all_concept_weights[concept]
k_to_entropy.append((k, entropy))
return max(k_to_entropy, key=lambda x: x[1])[0]
def set_k_by_L(self):
# TODO sequentially
for k in range(1, self.get_corpus_size() + 1):
bigram_count = len(
set([
c for sent in self.get_top_k_sentences(k)
for c in sent.concepts
]))
# We count unique bigrams, hence / 2
if bigram_count / 2 >= self.summary_length:
break
return k
def get_baseline_entropies(self):
# Max, min baselines
print(self.k_to_entropy)
return [
m(self.k_to_entropy, key=lambda x: x[1])[0] for m in [max, min]
]
def get_number_of_concepts(self):
return self.num_of_concepts_in_summary
# Time based strategy #
def set_k_by_target_time(self):
# TODO parametrize max_time
max_time = 1
target_constraint_size = int(self.time_to_ilp_constraints(max_time))
occurences = 0
concepts = set()
k = 0
while ((occurences + len(concepts) + 1) < target_constraint_size
or len(concepts) < self.summary_length / 2):
sent_id = self.ranks_to_sentences.iloc[k]
k += 1
for c in set(self.sentences_dict[sent_id].concepts):
occurences += 1
concepts.add(c)
return k
def k_to_constraint_size(self, k):
if type(k) is pd.core.series.Series:
s = {}
se = []
unique_k = set(k)
for i in unique_k:
# s.append(self.get_constraint_size_for(i))
s[i] = self.get_constraint_size_for(i)
for i in k:
se.append(s[i])
return pd.Series(se)
else:
return self.get_constraint_size_for(int(k))
def time_to_ilp_constraints(self, t):
return self.cost_model.constraints(t)
def get_constraint_size_for(self, k):
top_k_sent_ids = self.get_sentence_ids_for(k)
occurences = 0
concepts = set()
for s_id in top_k_sent_ids:
for ci in set(self.sentences_dict[s_id].concepts):
occurences += 1
concepts.add(ci)
return occurences + len(concepts) + 1
# Adaptive Window #
def get_important_sentences(self):
num_of_important_sents = 0
# Get number of consecutive sentences that have at least one important concept
for i, (sent_id,
metric_value) in enumerate(self.ranks_to_sentences.items()):
for concept in self.sentences_dict[sent_id].concepts:
if concept in self.important_concepts:
num_of_important_sents += 1
break
if (i + 1) != num_of_important_sents:
break
return int(num_of_important_sents * (1 + self.adaptive_window_size))
# Redundancy Sweep #
def get_distinct_top_k_sentences(self):
top_k_sent_ids = list(self.ranks_to_sentences.keys())
distinct_sentences = []
seen_concepts = defaultdict(lambda: False)
i = 0
while len(distinct_sentences) < self.k and i < self.get_corpus_size():
skip = False
sent = self.sentences_dict[top_k_sent_ids[i]]
counter = 0
for c in set(sent.concepts):
if seen_concepts[c]:
# +1 -> threshold = 0 means no overlap; 1 means one concept-overlap allowed
if counter >= self.sweep_threshold + 1:
skip = True
break
counter += 1
if not skip:
distinct_sentences.append(sent)
for c in sent.concepts:
seen_concepts[c] = 1
i += 1
return distinct_sentences
def get_sents_with_accepted_concepts(self):
input_sentences = self.get_top_k_sentences()
for sent_id in self.ranks_to_sentences.iloc[self.k:]:
sent = self.sentences_dict[sent_id]
for c in sent.concepts:
if c in self.important_concepts:
input_sentences.append(sent)
break
return input_sentences
def get_top_k_sentences(self, k=None):
print('### Top k %f' % self.k)
if k is None:
k = self.k
# Statistic purposes:
top_k_sent_ids = [
sent_id for sent_id in self.ranks_to_sentences.iloc[:k]
]
self.seen_sentences |= set(top_k_sent_ids)
return [
self.sentences_dict[sent_id]
for sent_id in self.ranks_to_sentences.iloc[:k]
]
def get_top_k_sentence_ids(self):
return self.ranks_to_sentences.iloc[:self.k]
def get_sentence_ids_for(self, k):
return self.ranks_to_sentences.iloc[:k]
def bisect_rank_by_value(self, value):
'''workaround method as ValueSortedDict bisects by key (not sensible for sort order by value).'''
self.ranks_to_sentences["bisect"] = value
index = self.ranks_to_sentences.index("bisect")
del self.ranks_to_sentences["bisect"]
return index
def get_corpus_size(self):
return len(self.sentences_dict)
# following: stuff for test purposes
def rank_to_metric(self, rank):
return self.ranks_to_sentences[self.ranks_to_sentences.iloc[rank]]
def print_ranked_sentences(self, n=10, full_sentence=False):
for rank, (doc_id,
position) in enumerate(self.get_top_k_sentence_ids()[:n],
1):
print("Rank: ", rank)
if full_sentence:
# self.sentences_dict[sent_id] returns the desired sentence
print(self.sentences_dict[(doc_id, position)].untokenized_form)
else:
print("doc_id: {}, sentence_pos: {}".format(doc_id, position))
# self.ranks_to_sentences[sent_id] returns the metric
print("Heuristic value: ",
self.ranks_to_sentences[(doc_id, position)])
print("#-----#")
def print_concept_map(self):
i = 0
for key, value in self.concept_to_sentences.items():
print(key, ":", self.concept_to_sentences[key])
for entry in self.concept_to_sentences[key]:
print(self.sentences_dict[entry].untokenized_form)
i += 1
print("-------")
if i >= 10:
break
class Field(mp.Process):
"""Represents the detected playing field and it's contents. Also handles path creation."""
def __init__(self, id, car, gui):
super(Field, self).__init__(target=self.dispatch, name='Field_{}_proc'.format(id))
self.exit = mp.Event()
self.id = id
self.car = car
self.gui = gui # type: FieldGUI
self.width = 150
self.height = 90
self.houses = dict()
self.xes = ValueSortedDict()
self.houseupdates = ValueSortedDict() # Essentially a priority queue with more functionality
self.update_idx = 0
self.next_new_house = 0
self.potentials = np.zeros((H, W), dtype = float)
self.que = mp.Queue(4)
self.path = list()
self.start()
def stop(self):
self.exit.set()
def dispatch(self):
while not self.exit.is_set():
try:
func, args, kwargs = self.que.get(timeout=1)
except Queue.Empty:
pass
else:
type(self).__dict__[func](self, *args, **kwargs)
def reset(self):
"""Invalidate static info."""
self.call('_reset')
def _reset(self):
self.houses.clear()
self.houseupdates.clear()
self.xes.clear()
self.update_idx = 0
self.next_new_house = 0
def call(self, func, *args, **kwargs):
self.que.put((func, args, kwargs)) # TODO: Change to put_nowait before the competition
def update_houses(self, housearray):
self.call('_update_houses', housearray)
def _update_houses(self, housearray):
self.update_idx += 1
for ha in housearray:
# Get an iterator over all houses that have center x coordinate within 5 cm of the input value.
# Then see if one of the houses is close enough to be considered the same. If there are, update it's
# location
point = ha[2:4]
to_change = None
for oldhouse_idx in self.xes.irange_key(point[0] - D, point[0] + D): # in cm.
oldhouse = self.houses[oldhouse_idx]
if -D < oldhouse.center.coords()[1] - point[1] < D and oldhouse.color == ha[1]:
to_change = oldhouse_idx
break
if to_change is not None:
house = self.houses[to_change]
house.center.update(point)
else:
to_change = self.next_new_house
self.next_new_house += 1
house = House(point, ha[1])
self.xes[to_change] = house.center.coords()[0]
self.houses[to_change] = house
self.houseupdates[to_change] = self.update_idx
# if not self.update_idx % 5:
self.clean_houses()
self.update_potentials()
if self.gui.enabled:
self.draw_houses()
self.draw_path(self._create_path((45, 75), self.car.yxintcoords()))
def update_potentials(self):
self.potentials = np.zeros((H, W), np.float32)
for house in self.houses.values(): # type: House
if house.color == House.RED:
weight = R_WEIGHT
elif house.color == House.GREEN:
weight = G_WEIGHT
else:
weight = D_WEIGHT
x, y = house.center.coords()
self.potentials[int(y - 2 + 0.5):int(y + 2 + 0.5), int(x - 2 + 0.5):int(x + 2 + 0.5)] = weight
cv2.GaussianBlur(self.potentials, (K_SIZE, K_SIZE), SIGMA, self.potentials, borderType=cv2.BORDER_CONSTANT)
np.maximum(self.potentials, 0, self.potentials)
def neighbours(self, point):
"""Return list of neighbour cells. 8 neighbours"""
y, x = point
ret = list()
if y > 0:
if x > 0:
ret.append((y - 1, x - 1))
ret.append((y - 1, x))
if x < W - 1:
ret.append((y - 1, x + 1))
if x > 0:
ret.append((y, x - 1))
if x < W-1:
ret.append((y, x + 1))
if y < H - 1:
if x > 0:
ret.append((y + 1, x - 1))
ret.append((y + 1, x))
if x < W - 1:
ret.append((y + 1, x + 1))
return ret
def create_path(self, start, end):
self.call('_create_path', start, end)
def _create_path(self, start, end):
if H <= start[0] or W <= start[1] or \
H <= end[0] or W <= end[1] or \
start[0] < 0 or start[1] < 0 or \
end[0] < 0 or end[1] < 0:
return []
from math import sqrt
pq = SortedList(key=lambda x: -x[0]) # Priority Queue, with priority as the first element of entries
parents = dict()
parents[start] = None
dist = dict()
dist[start] = 0
# The cost function is straight-line distance + needed climb.
def costfunc(point, other):
stress = self.potentials[other] - self.potentials[point]
if stress < 0:
stress *= 0.8
pot = stress
dist = 1. if point[0] == other[0] or point[1] == other[1] else 1.414
return pot + dist
# The heuristic is straight-line distance plus needed climb
def heuristic(point):
pot = max(self.potentials[end] - self.potentials[point], 0)
# pot = 0 # Might cause problems where it prefers to go around green houses when green houses have negative weight.
dist = sqrt((end[0] - point[0]) ** 2 + (end[1] - point[1]) ** 2)
return pot + dist
pq.add((heuristic(start), start, None))
while True:
try:
est_u, u, p = pq.pop()
parents[u] = p
except IndexError:
print('No path found.')
u = None
break
if u == end:
break
for v in self.neighbours(u):
new_dist = dist[u] + costfunc(u, v)
if v not in dist or new_dist < dist[v]:
dist[v] = new_dist
est_v = new_dist + heuristic(v)
pq.add((est_v, v, u))
path = []
while u is not None:
path.append(u)
u = parents[u]
path.reverse()
self.path = path
return path
def draw_path(self, path):
if path:
path = np.array(path)
path = path.T
img = np.frombuffer(self.gui.im_array.get_obj(), np.uint8).reshape((H, W, 3))
img[path[0], path[1], :] = [255, 255, 200]
def draw_houses(self):
frame = np.frombuffer(self.gui.im_array.get_obj(), np.uint8, H * W * 3).reshape((H, W, 3))
frame[:] = cv2.cvtColor(cv2.convertScaleAbs(self.potentials, alpha=1), cv2.COLOR_GRAY2BGR)
for house in self.houses.values():
x, y = house.center.coords()
xl, yl = max(x - 2, 0), max(y - 2, 0)
xh, yh = min(x + 2, W - 1), min(y + 2, H - 1)
if house.color == House.RED:
col = (0, 0, 255)
elif house.color == House.GREEN:
col = (0, 255, 0)
frame[yl:yh, xl:xh] = col
if self.path:
self.draw_path(self.path)
def clean_houses(self):
"""Clean up houses that haven't been updated in 5 or more turns."""
indices = list(self.houseupdates.irange_key(None, self.update_idx - 5))
for idx in indices:
del self.xes[idx]
del self.houses[idx]
del self.houseupdates[idx]
