def test_init_args(): temp = ValueSortedDict(enumerate(alphabet)) assert len(temp) == 26 assert temp[0] == 'a' assert temp[25] == 'z' assert temp.iloc[4] == 4 temp._check()
def initialize(self, parent=None): # one model per cluster in current_state self._models = {} for cluster in parent.current_state.labels(): self._models[cluster] = self.compute_model(cluster, parent=parent) # list of clusters clusters = list(self._models) try: self._similarity = self.compute_similarity_matrix(parent=parent) except NotImplementedError as e: n_clusters = len(clusters) self._similarity = ValueSortedDict() for i, j in combinations(clusters, 2): # compute similarity if (and only if) clusters are mergeable if not parent.constraint.mergeable([i, j], parent=parent): self._similarity[i, j] = -np.inf self._similarity[j, i] = -np.inf continue similarity = self.compute_similarity(i, j, parent=parent) self._similarity[i, j] = similarity if not self.is_symmetric: similarity = self.compute_similarity(j, i, parent=parent) self._similarity[j, i] = similarity
def initialize_sentences(self, sentences, concept_weights): '''Initializes sentences based on metric "concept density". creates: sentences_dict, concept_to_sentences, ranks_to_sentences ''' self.sentences_dict = {} self.concept_to_sentences = defaultdict(set) # Highest value --> first rank self.ranks_to_sentences = ValueSortedDict(lambda x: -x) for sent in sentences: # Create sentences_dict sent_id = (sent.doc_id, sent.position) self.sentences_dict[sent_id] = sent # Calculate concept density per sentence concept_density = 0 for concept in sent.concepts: concept_density += concept_weights[concept] # Create concept2sent dic self.concept_to_sentences[concept].add(sent_id) concept_density /= float(sent.length) # create ranks_to_sentences self.ranks_to_sentences[sent_id] = concept_density
async def startup(self, app): self.app["highscore"] = {variant: ValueSortedDict(neg) for variant in VARIANTS} self.app["highscore"]["crazyhouse960"] = ValueSortedDict(neg, ZH960) self.wplayer = User(self.app, username="******", perfs=PERFS["user7"]) self.bplayer = User(self.app, username="******", perfs=PERFS["newplayer"]) self.strong_player = User(self.app, username="******", perfs=PERFS["strongplayer"]) self.weak_player = User(self.app, username="******", perfs=PERFS["weakplayer"])
def __init__(self, vertices: Iterable[any], compute_fills=True, compute_degrees=True): """ Create a Primal graph representing the interactions between vertices. For min-fill or min-induced-width, use remove_and_process_node() instead of use remove_node() :param vertices: The vertices of the graph :param compute_fills; Whether this will be used to compute fills. :param compute_degrees: Whether this will be used to compute degrees. """ self.connected_to: Dict[any, Set[any]] = {vertex: set() for vertex in vertices} self._fills: ValueSortedDict[any, Tuple[int, List[Tuple[any, Set[any]]]]] = ValueSortedDict(lambda n: n[0]) if \ compute_fills else None self._degrees: ValueSortedDict[any, int] = ValueSortedDict() if compute_degrees else None
def update_cpu_burn(name, count, t, l): burn = burners.get(name, {}) burn['count'] = burn.get('count', 0) + count burn['time'] = burn.get('time', 0.0) + t if l is not None: l = ValueSortedDict(l) burn['list'] = burn.get('list', ValueSortedDict()) for k in l: # XXX replace this loop with .update() burn['list'][k] = l[k] length = len(burn['list']) for _ in range(10, length): burn['list'].popitem() burners[name] = burn
async def create_new_pairings(self, waiting_players): pairing = self.create_pairing(waiting_players) if self.first_pairing: self.first_pairing = False # Before tournament starts leaderboard is ordered by ratings # After first pairing it will be sorted by score points and performance # so we have to make a clear (all 0) leaderboard here new_leaderboard = [(user, 0) for user in self.leaderboard] self.leaderboard = ValueSortedDict(neg, new_leaderboard) self.leaderboard_keys_view = SortedKeysView(self.leaderboard) games = await self.create_games(pairing) return (pairing, games)
def load_scores(self) -> None: """ Loads all of the scores from the flairs on Reddit. """ self.scores = ValueSortedDict({}) for flair in self.bot.reddit.main_subreddit.flair(limit=None): try: self.scores[flair["user"].name.lower()] = int( "".join([char for char in flair["flair_text"].split(" ")[0] if char.isnumeric()]) ) except Exception as e: print(e) pass print("POINTS: Loaded scores.")
class FluidStreamDiscretizer(): def __init__(self, bin_count, history_length): self.history = ValueSortedDict() self.step = 0 self.thresholds = np.linspace(0, 1, bin_count - 1) self.history_length = history_length self.saturated = False def __call__(self, value): self.step += 1 self.history[self.step] = value values = np.array(self.history.values()) bin_count = len(self.thresholds) + 1 self.thresholds = values[[ floor(idx * len(self.history) / bin_count) for idx in range(1, bin_count) ]] try: del self.history[self.step - self.history_length] self.saturated = True except KeyError: pass return np.digitize(value, self.thresholds)
def __init__(self, palettes: List[OrderedSet], num_palettes: int, colors_per_palette: int): """ :param palettes: A list of ordered sets of palettes :param palette_relations: A list of lists of possible palettes for tiles :param num_palettes: The number of palettes to reduce down to :param colors_per_palette: The number of colors per palette """ self.was_run = False self._palettes = [p.copy() for p in palettes] self._current_number_of_palettes = len(palettes) self._num_palettes = num_palettes self._colors_per_palette = colors_per_palette self._merges_performed: List[Tuple[int, int]] = [] self._count_per_pal = ValueSortedDict( {pidx: -len(p) for pidx, p in enumerate(self._palettes)})
def __init__(self, id, car, gui): super(Field, self).__init__(target=self.dispatch, name='Field_{}_proc'.format(id)) self.exit = mp.Event() self.id = id self.car = car self.gui = gui # type: FieldGUI self.width = 150 self.height = 90 self.houses = dict() self.xes = ValueSortedDict() self.houseupdates = ValueSortedDict() # Essentially a priority queue with more functionality self.update_idx = 0 self.next_new_house = 0 self.potentials = np.zeros((H, W), dtype = float) self.que = mp.Queue(4) self.path = list() self.start()
def __init__(self, fname, restart=False): if '/' not in fname: fname = os.path.join(DIR, fname) if not os.path.exists(os.path.dirname(fname)): os.makedirs(os.path.dirname(fname)) self.fname = fname print "Loading Recorder from", self.fname self.record = ValueSortedDict(lambda x: -x[0]) self.write_freq = 10 self.read(restart) self.write()
def identity_words(df): # Analyze words, derive stats good_counter = Counter() good_count = 0 for batch in range(int(len(df) / 100000)): sum_identities = np.sum(df.iloc[(batch) * 100000:(batch + 1) * 100000][REAL_IDENTITY_COLUMNS].values, axis=1).astype('bool') good_comments = ' '.join(df.iloc[(batch) * 100000:(batch + 1) * 100000][sum_identities == 0].apply( lambda row: row['comment_text'], axis=1)).split(' ') good_counter.update(good_comments) good_count += len(good_comments) del good_comments sum_identities = np.sum(df[REAL_IDENTITY_COLUMNS].values, axis=1).astype('bool') foul_comments = ' '.join(df[sum_identities > 0].apply( lambda row: row['comment_text'], axis=1)).split(' ') foul_counter = Counter(foul_comments) foul_count = len(foul_comments) del foul_comments word_foulness = ValueSortedDict() for word, count in good_counter.most_common() + foul_counter.most_common(): if count > MIN_FOUL_COUNT: bad_freq = foul_counter[word] / foul_count good_freq = (good_counter[word] + 1) / good_count bad_prob = bad_freq / (good_freq) word_foulness[word + '_'] = bad_prob # Because of a weird bug we had to add an underscore to every word, let's remove it foul_words = [ word[:-1] for word in list( word_foulness.islice(word_foulness.bisect_key(FOUL_FREQ_TO_DROP))) ] return foul_words
def knn_search(self, q=None, k=1): self.q = q self.visitedset = set() self.candidates = ValueSortedDict() self.result = ValueSortedDict() count = 0 for i in range(self.m): v_ep = self.corpus[np.random.choice(list(self.corpus.keys()))] if self.dmat is None: cost = self.switch_metric(self.q.values, v_ep.values) else: cost = self.dmat[q.index][v_ep.index] count += 1 self.candidates[v_ep.index] = cost self.visitedset.add(v_ep.index) tempres = ValueSortedDict() while True: # get element c closest from candidates to q, and remove c # from candidates if len(self.candidates) > 0: c = self.get_closest() else: break # check stop condition if len(self.result) >= k: if self.check_stop_condition(c, k): break tempres.update(c) # add neighbors of c if not in visitedset c = self.corpus[list(c.keys())[0]] for key in list(c.neighbors.keys()): if key not in self.visitedset: if self.dmat is None: cost = self.switch_metric(self.q.values, v_ep.values) else: cost = self.dmat[q.index][v_ep.index] count += 1 self.visitedset.add(key) self.candidates[key] = cost tempres[key] = cost # add tempres to result self.result.update(tempres) # return k neighbors/result return self.result, count
def setUp(self): self.loop = asyncio.get_event_loop() self.app = {} self.app["db"] = None self.app["users"] = {} self.app["games"] = {} self.app["tasks"] = weakref.WeakSet() self.app["crosstable"] = {} self.app["highscore"] = { variant: ValueSortedDict(neg) for variant in VARIANTS } self.app["highscore"]["crazyhouse960"] = ValueSortedDict(neg, ZH960) self.wplayer = User(self.app, username="******", perfs=PERFS["Doooovid"]) self.bplayer = User(self.app, username="******", perfs=PERFS["pepellou"]) self.splayer = User(self.app, username="******", perfs=PERFS["strongplayer"])
class Node: def __init__(self, index: int, values: list, label=None): self.index = index self.values = values self.label = label self.neighbors = ValueSortedDict() def __repr__(self): return {'index': self.index, 'label': self.label} def __str__(self): return 'Node(index=' + str(self.index) + ', Label=' + str( self.label) + ')' def connect(self, index, cost, f): """ Calculate distance and store in a sorteddict """ # The dict would be sorted by values self.neighbors[index] = cost while len(self.neighbors) > f: self.neighbors.popitem() return self
def compute_similarity_matrix(self, parent=None): clusters = list(self._models) n_clusters = len(clusters) X = np.vstack([self[cluster][0] for cluster in clusters]) nX = l2_normalize(X) similarities = -squareform(pdist(nX, metric=self.distance)) matrix = ValueSortedDict() for i, j in itertools.combinations(range(n_clusters), 2): matrix[clusters[i], clusters[j]] = similarities[i, j] matrix[clusters[j], clusters[i]] = similarities[j, i] return matrix
def compute_similarities(self, cluster, clusters, parent=None): x = self[cluster][0].reshape((1, -1)) X = np.vstack([self[c][0] for c in clusters]) # L2 normalization nx = l2_normalize(x) nX = l2_normalize(X) similarities = -cdist(nx, nX, metric=self.distance) matrix = ValueSortedDict() for i, cluster_ in enumerate(clusters): matrix[cluster, cluster_] = similarities[0, i] matrix[cluster_, cluster] = similarities[0, i] return matrix
def record_a_burn(name, start, url=None): if isinstance(url, URL): url = url.url elapsed = time.process_time() - start burn = burners.get(name, {}) burn['count'] = burn.get('count', 0) + 1 burn['time'] = burn.get('time', 0.0) + elapsed avg = burn.get('avg', 10000000.) # are we exceptional? 10x current average and significant if elapsed > avg * 10 and elapsed > 0.015: if 'list' not in burn: burn['list'] = ValueSortedDict() url = url or 'none' burn['list'][url] = -elapsed length = len(burn['list']) for _ in range(10, length): burn['list'].popitem() burn['avg'] = burn['time'] / burn['count'] burners[name] = burn
def record_a_latency(name, start, url=None, elapsedmin=10.0): if isinstance(url, URL): url = url.url elapsed = time.time() - start latency = latencies.get(name, {}) latency['count'] = latency.get('count', 0) + 1 latency['time'] = latency.get('time', 0.0) + elapsed if 'hist' not in latency: latency['hist'] = HdrHistogram(1, 30 * 1000, 2) # 1ms-30sec, 2 sig figs latency['hist'].record_value(elapsed * 1000) # ms if elapsed > elapsedmin: if 'list' not in latency: latency['list'] = ValueSortedDict() url = url or 'none' length = len(latency['list']) if length > 9: for u in itertools.islice(latency['list'], 9, length): del latency['list'][u] latency['list'][url] = -elapsed latencies[name] = latency
def compute_similarity_matrix(self, parent=None): # name and number of clusters clusters = list(self._models) n_clusters = len(clusters) # precompute pairwise embedding distance data = parent.features X = np.array(data[data.columns[2:]]) self.precomputed_ = -squareform(pdist(X, metric='euclidean')) matrix = ValueSortedDict() for i, j in itertools.combinations(range(n_clusters), 2): # indices of embedding in ith cluster indices_i = self[clusters[i]] # indices of embedding in jth cluster indices_j = self[clusters[j]] # mean of all pairwise euclidean distances similarity = np.mean(self.precomputed_[indices_i][:, indices_j]) matrix[clusters[i], clusters[j]] = similarity matrix[clusters[j], clusters[i]] = similarity return matrix
def record_a_latency(name, start, url=None, elapsedmin=10.0): if isinstance(url, URL): url = url.url elapsed = time.time() - start latency = latencies.get(name, {}) latency['count'] = latency.get('count', 0) + 1 latency['time'] = latency.get('time', 0.0) + elapsed if 'hist' not in latency: latency['hist'] = HdrHistogram(1, 30 * 1000, 2) # 1ms-30sec, 2 sig figs latency['hist'].record_value(elapsed * 1000) # ms # show the 10 most recent latencies > 10 seconds if elapsed > elapsedmin: if 'list' not in latency: latency['list'] = ValueSortedDict() url = url or 'none' length = len(latency['list']) for _ in range(9, length): latency['list'].popitem( last=False) # throwing away biggest value(s) latency['list'][url] = -elapsed latencies[name] = latency
def test_init(): temp = ValueSortedDict() temp._check()
def boring_to_burners(d): global burners for k in d: burners[k] = d[k] burners[k]['list'] = ValueSortedDict(d[k].get('list', dict()))
def cluster_(self, fX): """Compute complete dendrogram Parameters ---------- fX : (n_items, dimension) np.array Embeddings. Returns ------- dendrogram : list of (i, j, distance) tuples Dendrogram. """ N = len(fX) # clusters contain the identifier of each cluster clusters = SortedSet(np.arange(N)) # labels[i] = c means ith item belongs to cluster c labels = np.array(np.arange(N)) squared = squareform(pdist(fX, metric=self.metric)) distances = ValueSortedDict() for i, j in itertools.combinations(range(N), 2): distances[i, j] = squared[i, j] dendrogram = [] for _ in range(N-1): # find most similar clusters (c_i, c_j), d = distances.peekitem(index=0) # keep track of this iteration dendrogram.append((c_i, c_j, d)) # index of clusters in 'clusters' and 'fX' i = clusters.index(c_i) j = clusters.index(c_j) # merge items of cluster c_j into cluster c_i labels[labels == c_j] = c_i # update c_i representative fX[i] += fX[j] # remove c_j cluster fX[j:-1, :] = fX[j+1:, :] fX = fX[:-1] # remove distances to c_j cluster for c in clusters[:j]: distances.pop((c, c_j)) for c in clusters[j+1:]: distances.pop((c_j, c)) clusters.remove(c_j) if len(clusters) < 2: continue # compute distance to new c_i cluster new_d = cdist(fX[i, :].reshape((1, -1)), fX, metric=self.metric).squeeze() for c_k, d in zip(clusters, new_d): if c_k < c_i: distances[c_k, c_i] = d elif c_k > c_i: distances[c_i, c_k] = d return dendrogram
async def init_state(app): # We have to put "kill" into a dict to prevent getting: # DeprecationWarning: Changing state of started or joined application is deprecated app["data"] = {"kill": False} if "db" not in app: app["db"] = None app["users"] = { "Random-Mover": User(app, bot=True, username="******"), "Fairy-Stockfish": User(app, bot=True, username="******"), "Discord-Relay": User(app, anon=True, username="******"), } app["users"]["Random-Mover"].online = True app["lobbysockets"] = {} app["seeks"] = {} app["games"] = {} app["invites"] = {} app["chat"] = collections.deque([], 100) app["game_channels"] = set() app["invite_channels"] = set() app["highscore"] = {variant: ValueSortedDict(neg) for variant in VARIANTS} app["crosstable"] = {} app["stats"] = {} # counters for games app["g_cnt"] = 0 # last game played app["tv"] = None # fishnet active workers app["workers"] = set() # fishnet works app["works"] = {} # fishnet worker tasks app["fishnet"] = asyncio.PriorityQueue() # fishnet workers monitor app["fishnet_monitor"] = {} app["fishnet_versions"] = {} for key in FISHNET_KEYS: app["fishnet_monitor"][FISHNET_KEYS[key]] = collections.deque([], 50) rm = app["users"]["Random-Mover"] for variant in VARIANTS: variant960 = variant.endswith("960") variant_name = variant[:-3] if variant960 else variant byoyomi = variant.endswith("shogi") or variant in ("dobutsu", "gorogoro", "janggi", "shogun") seek = Seek(rm, variant_name, base=5, inc=30 if byoyomi else 3, level=0, chess960=variant960, byoyomi_period=1 if byoyomi else 0) app["seeks"][seek.id] = seek rm.seeks[seek.id] = seek ai = app["users"]["Fairy-Stockfish"] asyncio.create_task(BOT_task(ai, app)) asyncio.create_task(BOT_task(rm, app)) # Configure templating. app["jinja"] = {} base = os.path.dirname(__file__) for lang in LANGUAGES: # Generate compiled mo file folder = os.path.join(base, "../lang/", lang, "LC_MESSAGES") poname = os.path.join(folder, "server.po") moname = os.path.join(folder, "server.mo") try: with open(poname, 'rb') as po_file: po_lines = [ line for line in po_file if line[:8] != b"#, fuzzy" ] mo = Msgfmt(po_lines).get() with open(moname, 'wb') as mo_file: mo_file.write(mo) except PoSyntaxError: log.error("PoSyntaxError in %s", poname) # Create translation class try: translation = gettext.translation("server", localedir="lang", languages=[lang]) except FileNotFoundError: log.warning("Missing translations file for lang %s", lang) translation = gettext.NullTranslations() env = jinja2.Environment(enable_async=True, extensions=['jinja2.ext.i18n'], loader=jinja2.FileSystemLoader("templates"), autoescape=jinja2.select_autoescape(["html"])) env.install_gettext_translations(translation, newstyle=True) env.globals["static"] = static_url app["jinja"][lang] = env if app["db"] is None: return # Read users and highscore from db try: cursor = app["db"].user.find() async for doc in cursor: if doc["_id"] not in app["users"]: perfs = doc.get("perfs") if perfs is None: perfs = {variant: DEFAULT_PERF for variant in VARIANTS} app["users"][doc["_id"]] = User( app, username=doc["_id"], title=doc.get("title"), first_name=doc.get("first_name"), last_name=doc.get("last_name"), country=doc.get("country"), bot=doc.get("title") == "BOT", perfs=perfs, enabled=doc.get("enabled", True)) db_collections = await app["db"].list_collection_names() if "highscore" not in db_collections: await generate_highscore(app["db"]) cursor = app["db"].highscore.find() async for doc in cursor: app["highscore"][doc["_id"]] = ValueSortedDict(neg, doc["scores"]) if "crosstable" not in db_collections: await generate_crosstable(app["db"]) cursor = app["db"].crosstable.find() async for doc in cursor: app["crosstable"][doc["_id"]] = doc await app["db"].game.create_index("us") await app["db"].game.create_index("v") await app["db"].game.create_index("y") await app["db"].game.create_index("by") except Exception: print("Maybe mongodb is not running...") raise
class Tournament(ABC): """Abstract base class for Arena/Swisss/RR Tournament classes They have to implement create_pairing() for waiting_players""" system: ClassVar[int] = ARENA def __init__( self, app, tournamentId, variant="chess", chess960=False, rated=True, before_start=5, minutes=45, name="", description="", fen="", base=1, inc=0, byoyomi_period=0, rounds=0, created_by="", created_at=None, starts_at=None, status=None, with_clock=True, frequency="", ): self.app = app self.id = tournamentId self.name = name self.description = description self.variant = variant self.rated = rated self.before_start = before_start # in minutes self.minutes = minutes # in minutes self.fen = fen self.base = base self.inc = inc self.byoyomi_period = byoyomi_period self.chess960 = chess960 self.rounds = rounds self.frequency = frequency self.created_by = created_by self.created_at = datetime.now(timezone.utc) if created_at is None else created_at if starts_at == "" or starts_at is None: self.starts_at = self.created_at + timedelta(seconds=int(before_start * 60)) else: self.starts_at = starts_at # TODO: calculate wave from TC, variant, number of players self.wave = timedelta(seconds=3) self.wave_delta = timedelta(seconds=1) self.current_round = 0 self.prev_pairing = None self.messages = collections.deque([], MAX_CHAT_LINES) self.spectators = set() self.players: dict[User, PlayerData] = {} self.leaderboard = ValueSortedDict(neg) self.leaderboard_keys_view = SortedKeysView(self.leaderboard) self.status = T_CREATED if status is None else status self.ongoing_games = 0 self.nb_players = 0 self.nb_games_finished = 0 self.w_win = 0 self.b_win = 0 self.draw = 0 self.nb_berserk = 0 self.nb_games_cached = -1 self.leaderboard_cache = {} self.first_pairing = False self.top_player = None self.top_game = None self.notify1 = False self.notify2 = False if minutes is None: self.ends_at = self.starts_at + timedelta(days=1) else: self.ends_at = self.starts_at + timedelta(minutes=minutes) if with_clock: self.clock_task = asyncio.create_task(self.clock()) def __repr__(self): return " ".join((self.id, self.name, self.created_at.isoformat())) @abstractmethod def create_pairing(self, waiting_players): pass def user_status(self, user): if user in self.players: return ( "paused" if self.players[user].paused else "withdrawn" if self.players[user].withdrawn else "joined" ) else: return "spectator" def user_rating(self, user): if user in self.players: return self.players[user].rating else: return "%s%s" % user.get_rating(self.variant, self.chess960).rating_prov def players_json(self, page=None, user=None): if (page is None) and (user is not None) and (user in self.players): if self.players[user].page > 0: page = self.players[user].page else: div, mod = divmod(self.leaderboard.index(user) + 1, 10) page = div + (1 if mod > 0 else 0) if self.status == T_CREATED: self.players[user].page = page if page is None: page = 1 if self.nb_games_cached != self.nb_games_finished: # number of games changed (game ended) self.leaderboard_cache = {} self.nb_games_cached = self.nb_games_finished elif user is not None: if self.status == T_STARTED: # player status changed (JOIN/PAUSE) if page in self.leaderboard_cache: del self.leaderboard_cache[page] elif self.status == T_CREATED: # number of players changed (JOIN/WITHDRAW) self.leaderboard_cache = {} if page in self.leaderboard_cache: return self.leaderboard_cache[page] def player_json(player, full_score): return { "paused": self.players[player].paused if self.status == T_STARTED else False, "title": player.title, "name": player.username, "rating": self.players[player].rating, "points": self.players[player].points, "fire": self.players[player].win_streak, "score": full_score, # SCORE_SHIFT-ed + performance rating "perf": self.players[player].performance, "nbGames": self.players[player].nb_games, "nbWin": self.players[player].nb_win, "nbBerserk": self.players[player].nb_berserk, } start = (page - 1) * 10 end = min(start + 10, self.nb_players) page_json = { "type": "get_players", "requestedBy": user.username if user is not None else "", "nbPlayers": self.nb_players, "nbGames": self.nb_games_finished, "page": page, "players": [ player_json(player, full_score) for player, full_score in self.leaderboard.items()[start:end] ], } if self.status > T_STARTED: page_json["podium"] = [ player_json(player, full_score) for player, full_score in self.leaderboard.items()[0:3] ] self.leaderboard_cache[page] = page_json return page_json # TODO: cache this def games_json(self, player_name): player = self.app["users"].get(player_name) return { "type": "get_games", "rank": self.leaderboard.index(player) + 1, "title": player.title, "name": player_name, "perf": self.players[player].performance, "nbGames": self.players[player].nb_games, "nbWin": self.players[player].nb_win, "nbBerserk": self.players[player].nb_berserk, "games": [game.game_json(player) for game in self.players[player].games], } @property def spectator_list(self): return spectators(self) @property def top_game_json(self): return { "type": "top_game", "gameId": self.top_game.id, "variant": self.top_game.variant, "fen": self.top_game.board.fen, "w": self.top_game.wplayer.username, "b": self.top_game.bplayer.username, "wr": self.leaderboard_keys_view.index(self.top_game.wplayer) + 1, "br": self.leaderboard_keys_view.index(self.top_game.bplayer) + 1, "chess960": self.top_game.chess960, "base": self.top_game.base, "inc": self.top_game.inc, "byoyomi": self.top_game.byoyomi_period, } def waiting_players(self): return [ p for p in self.leaderboard if self.players[p].free and self.id in p.tournament_sockets and len(p.tournament_sockets[self.id]) > 0 and not self.players[p].paused and not self.players[p].withdrawn ] async def clock(self): try: while self.status not in (T_ABORTED, T_FINISHED, T_ARCHIVED): now = datetime.now(timezone.utc) if self.status == T_CREATED: remaining_time = self.starts_at - now remaining_mins_to_start = int( ((remaining_time.days * 3600 * 24) + remaining_time.seconds) / 60 ) if now >= self.starts_at: if self.system != ARENA and len(self.players) < 3: # Swiss and RR Tournaments need at least 3 players to start await self.abort() print("T_ABORTED: less than 3 player joined") break await self.start(now) continue elif (not self.notify2) and remaining_mins_to_start <= NOTIFY2_MINUTES: self.notify1 = True self.notify2 = True await discord_message( self.app, "notify_tournament", self.notify_discord_msg(remaining_mins_to_start), ) continue elif (not self.notify1) and remaining_mins_to_start <= NOTIFY1_MINUTES: self.notify1 = True await discord_message( self.app, "notify_tournament", self.notify_discord_msg(remaining_mins_to_start), ) continue elif (self.minutes is not None) and now >= self.ends_at: await self.finish() print("T_FINISHED: no more time left") break elif self.status == T_STARTED: if self.system == ARENA: # In case of server restart if self.prev_pairing is None: self.prev_pairing = now - self.wave if now >= self.prev_pairing + self.wave + random.uniform( -self.wave_delta, self.wave_delta ): waiting_players = self.waiting_players() nb_waiting_players = len(waiting_players) if nb_waiting_players >= 2: log.debug("Enough player (%s), do pairing", nb_waiting_players) await self.create_new_pairings(waiting_players) self.prev_pairing = now else: log.debug( "Too few player (%s) to make pairing", nb_waiting_players, ) else: log.debug("Waiting for new pairing wave...") elif self.ongoing_games == 0: if self.current_round < self.rounds: self.current_round += 1 log.debug("Do %s. round pairing", self.current_round) waiting_players = self.waiting_players() await self.create_new_pairings(waiting_players) else: await self.finish() log.debug("T_FINISHED: no more round left") break else: print( "%s has %s ongoing game(s)..." % ( "RR" if self.system == RR else "Swiss", self.ongoing_games, ) ) log.debug("%s CLOCK %s", self.id, now.strftime("%H:%M:%S")) await asyncio.sleep(1) except Exception: log.exception("Exception in tournament clock()") async def start(self, now): self.status = T_STARTED self.first_pairing = True self.set_top_player() response = { "type": "tstatus", "tstatus": self.status, "secondsToFinish": (self.ends_at - now).total_seconds(), } await self.broadcast(response) # force first pairing wave in arena if self.system == ARENA: self.prev_pairing = now - self.wave if self.app["db"] is not None: print( await self.app["db"].tournament.find_one_and_update( {"_id": self.id}, {"$set": {"status": self.status}}, return_document=ReturnDocument.AFTER, ) ) @property def summary(self): return { "type": "tstatus", "tstatus": self.status, "nbPlayers": self.nb_players, "nbGames": self.nb_games_finished, "wWin": self.w_win, "bWin": self.b_win, "draw": self.draw, "berserk": self.nb_berserk, "sumRating": sum( self.players[player].rating for player in self.players if not self.players[player].withdrawn ), } async def finalize(self, status): self.status = status if len(self.players) > 0: self.print_leaderboard() print("--- TOURNAMENT RESULT ---") for i in range(min(3, len(self.leaderboard))): player = self.leaderboard.peekitem(i)[0] print("--- #%s ---" % (i + 1), player.username) # remove latest games from players tournament if it was not finished in time for player in self.players: if len(self.players[player].games) == 0: continue latest = self.players[player].games[-1] if latest and latest.status in (CREATED, STARTED): self.players[player].games.pop() self.players[player].points.pop() self.players[player].nb_games -= 1 # force to create new players json data self.nb_games_cached = -1 await self.broadcast(self.summary) await self.save() await self.broadcast_spotlight() async def broadcast_spotlight(self): spotlights = tournament_spotlights(self.app["tournaments"]) lobby_sockets = self.app["lobbysockets"] response = {"type": "spotlights", "items": spotlights} await lobby_broadcast(lobby_sockets, response) async def abort(self): await self.finalize(T_ABORTED) async def finish(self): await self.finalize(T_FINISHED) async def join(self, user): if user.anon: return if self.system == RR and len(self.players) > self.rounds + 1: raise EnoughPlayer if user not in self.players: # new player joined rating, provisional = user.get_rating(self.variant, self.chess960).rating_prov self.players[user] = PlayerData(rating, provisional) elif self.players[user].withdrawn: # withdrawn player joined again rating, provisional = user.get_rating(self.variant, self.chess960).rating_prov if user not in self.leaderboard: # new player joined or withdrawn player joined again if self.status == T_CREATED: self.leaderboard.setdefault(user, rating) else: self.leaderboard.setdefault(user, 0) self.nb_players += 1 self.players[user].paused = False self.players[user].withdrawn = False response = self.players_json(user=user) await self.broadcast(response) if self.status == T_CREATED: await self.broadcast_spotlight() await self.db_update_player(user, self.players[user]) async def withdraw(self, user): self.players[user].withdrawn = True self.leaderboard.pop(user) self.nb_players -= 1 response = self.players_json(user=user) await self.broadcast(response) await self.broadcast_spotlight() await self.db_update_player(user, self.players[user]) async def pause(self, user): self.players[user].paused = True # pause is different from withdraw and join because pause can be initiated from finished games page as well response = self.players_json(user=user) await self.broadcast(response) if (self.top_player is not None) and self.top_player.username == user.username: self.set_top_player() await self.db_update_player(user, self.players[user]) def spactator_join(self, spectator): self.spectators.add(spectator) def spactator_leave(self, spectator): self.spectators.discard(spectator) async def create_new_pairings(self, waiting_players): pairing = self.create_pairing(waiting_players) if self.first_pairing: self.first_pairing = False # Before tournament starts leaderboard is ordered by ratings # After first pairing it will be sorted by score points and performance # so we have to make a clear (all 0) leaderboard here new_leaderboard = [(user, 0) for user in self.leaderboard] self.leaderboard = ValueSortedDict(neg, new_leaderboard) self.leaderboard_keys_view = SortedKeysView(self.leaderboard) games = await self.create_games(pairing) return (pairing, games) def set_top_player(self): idx = 0 self.top_player = None while idx < self.nb_players: top_player = self.leaderboard.peekitem(idx)[0] if self.players[top_player].paused: idx += 1 continue else: self.top_player = top_player break async def create_games(self, pairing): check_top_game = self.top_player is not None new_top_game = False games = [] game_table = None if self.app["db"] is None else self.app["db"].game for wp, bp in pairing: game_id = await new_id(game_table) game = Game( self.app, game_id, self.variant, self.fen, wp, bp, base=self.base, inc=self.inc, byoyomi_period=self.byoyomi_period, rated=RATED if self.rated else CASUAL, tournamentId=self.id, chess960=self.chess960, ) games.append(game) self.app["games"][game_id] = game await insert_game_to_db(game, self.app) # TODO: save new game to db if 0: # self.app["db"] is not None: doc = { "_id": game.id, "tid": self.id, "u": [game.wplayer.username, game.bplayer.username], "r": "*", "d": game.date, "wr": game.wrating, "br": game.brating, } await self.app["db"].tournament_pairing.insert_one(doc) self.players[wp].games.append(game) self.players[bp].games.append(game) self.players[wp].points.append("*") self.players[bp].points.append("*") self.ongoing_games += 1 self.players[wp].free = False self.players[bp].free = False self.players[wp].nb_games += 1 self.players[bp].nb_games += 1 self.players[wp].prev_opp = game.bplayer.username self.players[bp].prev_opp = game.wplayer.username self.players[wp].color_balance += 1 self.players[bp].color_balance -= 1 self.players[wp].nb_not_paired = 0 self.players[bp].nb_not_paired = 0 response = { "type": "new_game", "gameId": game_id, "wplayer": wp.username, "bplayer": bp.username, } try: ws = next(iter(wp.tournament_sockets[self.id])) if ws is not None: await ws.send_json(response) except Exception: self.pause(wp) log.debug("White player %s left the tournament", wp.username) try: ws = next(iter(bp.tournament_sockets[self.id])) if ws is not None: await ws.send_json(response) except Exception: self.pause(bp) log.debug("Black player %s left the tournament", bp.username) if ( check_top_game and (self.top_player is not None) and self.top_player.username in (game.wplayer.username, game.bplayer.username) and game.status != BYEGAME ): # Bye game self.top_game = game check_top_game = False new_top_game = True if new_top_game: tgj = self.top_game_json await self.broadcast(tgj) return games def points_perfs(self, game: Game) -> Tuple[Point, Point, int, int]: wplayer = self.players[game.wplayer] bplayer = self.players[game.bplayer] wpoint = (0, SCORE) bpoint = (0, SCORE) wperf = game.black_rating.rating_prov[0] bperf = game.white_rating.rating_prov[0] if game.result == "1/2-1/2": if self.system == ARENA: if game.board.ply > 10: wpoint = (2, SCORE) if wplayer.win_streak == 2 else (1, SCORE) bpoint = (2, SCORE) if bplayer.win_streak == 2 else (1, SCORE) wplayer.win_streak = 0 bplayer.win_streak = 0 else: wpoint, bpoint = (1, SCORE), (1, SCORE) elif game.result == "1-0": wplayer.nb_win += 1 if self.system == ARENA: if wplayer.win_streak == 2: wpoint = (4, DOUBLE) else: wplayer.win_streak += 1 wpoint = (2, STREAK if wplayer.win_streak == 2 else SCORE) bplayer.win_streak = 0 else: wpoint = (2, SCORE) if game.wberserk and game.board.ply >= 13: wpoint = (wpoint[0] + 1, wpoint[1]) wperf += 500 bperf -= 500 elif game.result == "0-1": bplayer.nb_win += 1 if self.system == ARENA: if bplayer.win_streak == 2: bpoint = (4, DOUBLE) else: bplayer.win_streak += 1 bpoint = (2, STREAK if bplayer.win_streak == 2 else SCORE) wplayer.win_streak = 0 else: bpoint = (2, SCORE) if game.bberserk and game.board.ply >= 14: bpoint = (bpoint[0] + 1, bpoint[1]) wperf -= 500 bperf += 500 return (wpoint, bpoint, wperf, bperf) def points_perfs_janggi(self, game): wplayer = self.players[game.wplayer] bplayer = self.players[game.bplayer] wpoint = (0, SCORE) bpoint = (0, SCORE) wperf = game.black_rating.rating_prov[0] bperf = game.white_rating.rating_prov[0] if game.status == VARIANTEND: wplayer.win_streak = 0 bplayer.win_streak = 0 if game.result == "1-0": if self.system == ARENA: wpoint = (4 * 2 if wplayer.win_streak == 2 else 4, SCORE) bpoint = (2 * 2 if bplayer.win_streak == 2 else 2, SCORE) else: wpoint = (4, SCORE) bpoint = (2, SCORE) elif game.result == "0-1": if self.system == ARENA: bpoint = (4 * 2 if bplayer.win_streak == 2 else 4, SCORE) wpoint = (2 * 2 if wplayer.win_streak == 2 else 2, SCORE) else: bpoint = (4, SCORE) wpoint = (2, SCORE) elif game.result == "1-0": wplayer.nb_win += 1 if self.system == ARENA: if wplayer.win_streak == 2: wpoint = (7 * 2, DOUBLE) else: wplayer.win_streak += 1 wpoint = (7, STREAK if wplayer.win_streak == 2 else SCORE) bplayer.win_streak = 0 if game.wberserk and game.board.ply >= 13: wpoint = (wpoint[0] + 3, wpoint[1]) else: wpoint = (7, SCORE) bpoint = (0, SCORE) wperf += 500 bperf -= 500 elif game.result == "0-1": bplayer.nb_win += 1 if self.system == ARENA: if bplayer.win_streak == 2: bpoint = (7 * 2, DOUBLE) else: bplayer.win_streak += 1 bpoint = (7, STREAK if bplayer.win_streak == 2 else SCORE) wplayer.win_streak = 0 if game.bberserk and game.board.ply >= 14: bpoint = (bpoint[0] + 3, bpoint[1]) else: wpoint = (0, SCORE) bpoint = (7, SCORE) wperf -= 500 bperf += 500 return (wpoint, bpoint, wperf, bperf) async def game_update(self, game): """Called from Game.update_status()""" if self.status == T_FINISHED and self.status != T_ARCHIVED: return wplayer = self.players[game.wplayer] bplayer = self.players[game.bplayer] if game.wberserk: wplayer.nb_berserk += 1 self.nb_berserk += 1 if game.bberserk: bplayer.nb_berserk += 1 self.nb_berserk += 1 if game.variant == "janggi": wpoint, bpoint, wperf, bperf = self.points_perfs_janggi(game) else: wpoint, bpoint, wperf, bperf = self.points_perfs(game) wplayer.points[-1] = wpoint bplayer.points[-1] = bpoint if wpoint[1] == STREAK and len(wplayer.points) >= 2: wplayer.points[-2] = (wplayer.points[-2][0], STREAK) if bpoint[1] == STREAK and len(bplayer.points) >= 2: bplayer.points[-2] = (bplayer.points[-2][0], STREAK) wplayer.rating = game.white_rating.rating_prov[0] + (int(game.wrdiff) if game.wrdiff else 0) bplayer.rating = game.black_rating.rating_prov[0] + (int(game.brdiff) if game.brdiff else 0) # TODO: in Swiss we will need Berger instead of performance to calculate tie breaks nb = wplayer.nb_games wplayer.performance = int(round((wplayer.performance * (nb - 1) + wperf) / nb, 0)) nb = bplayer.nb_games bplayer.performance = int(round((bplayer.performance * (nb - 1) + bperf) / nb, 0)) wpscore = self.leaderboard.get(game.wplayer) // SCORE_SHIFT self.leaderboard.update( {game.wplayer: SCORE_SHIFT * (wpscore + wpoint[0]) + wplayer.performance} ) bpscore = self.leaderboard.get(game.bplayer) // SCORE_SHIFT self.leaderboard.update( {game.bplayer: SCORE_SHIFT * (bpscore + bpoint[0]) + bplayer.performance} ) self.nb_games_finished += 1 if game.result == "1-0": self.w_win += 1 elif game.result == "0-1": self.b_win += 1 elif game.result == "1/2-1/2": self.draw += 1 asyncio.create_task(self.delayed_free(game, wplayer, bplayer)) # TODO: save player points to db # await self.db_update_player(wplayer, self.players[wplayer]) # await self.db_update_player(bplayer, self.players[bplayer]) self.set_top_player() await self.broadcast( { "type": "game_update", "wname": game.wplayer.username, "bname": game.bplayer.username, } ) if self.top_game is not None and self.top_game.id == game.id: response = { "type": "gameEnd", "status": game.status, "result": game.result, "gameId": game.id, } await self.broadcast(response) if (self.top_player is not None) and self.top_player.username not in ( game.wplayer.username, game.bplayer.username, ): top_game_candidate = self.players[self.top_player].games[-1] if top_game_candidate.status != BYEGAME: self.top_game = top_game_candidate if (self.top_game is not None) and (self.top_game.status <= STARTED): tgj = self.top_game_json await self.broadcast(tgj) async def delayed_free(self, game, wplayer, bplayer): if self.system == ARENA: await asyncio.sleep(3) wplayer.free = True bplayer.free = True if game.status == FLAG: # pause players when they don't start their game if game.board.ply == 0: wplayer.paused = True elif game.board.ply == 1: bplayer.paused = True self.ongoing_games -= 1 async def broadcast(self, response): for spectator in self.spectators: try: for ws in spectator.tournament_sockets[self.id]: try: await ws.send_json(response) except ConnectionResetError: pass except KeyError: # spectator was removed pass except Exception: log.exception("Exception in tournament broadcast()") async def db_update_player(self, user, player_data): if self.app["db"] is None: return player_id = player_data.id player_table = self.app["db"].tournament_player if player_data.id is None: # new player join player_id = await new_id(player_table) player_data.id = player_id if player_data.withdrawn: new_data = { "wd": True, } else: full_score = self.leaderboard[user] new_data = { "_id": player_id, "tid": self.id, "uid": user.username, "r": player_data.rating, "pr": player_data.provisional, "a": player_data.paused, "f": player_data.win_streak == 2, "s": int(full_score / SCORE_SHIFT), "g": player_data.nb_games, "w": player_data.nb_win, "b": player_data.nb_berserk, "e": player_data.performance, "p": player_data.points, "wd": False, } try: print( await player_table.find_one_and_update( {"_id": player_id}, {"$set": new_data}, upsert=True, return_document=ReturnDocument.AFTER, ) ) except Exception: if self.app["db"] is not None: log.error( "db find_one_and_update tournament_player %s into %s failed !!!", player_id, self.id, ) new_data = {"nbPlayers": self.nb_players, "nbBerserk": self.nb_berserk} print( await self.app["db"].tournament.find_one_and_update( {"_id": self.id}, {"$set": new_data}, return_document=ReturnDocument.AFTER, ) ) async def save(self): if self.app["db"] is None: return if self.nb_games_finished == 0: print(await self.app["db"].tournament.delete_many({"_id": self.id})) print("--- Deleted empty tournament %s" % self.id) return winner = self.leaderboard.peekitem(0)[0].username new_data = { "status": self.status, "nbPlayers": self.nb_players, "nbGames": self.nb_games_finished, "winner": winner, } print( await self.app["db"].tournament.find_one_and_update( {"_id": self.id}, {"$set": new_data}, return_document=ReturnDocument.AFTER, ) ) pairing_documents = [] pairing_table = self.app["db"].tournament_pairing processed_games = set() for user, user_data in self.players.items(): for game in user_data.games: if game.status == BYEGAME: # ByeGame continue if game.id not in processed_games: pairing_documents.append( { "_id": game.id, "tid": self.id, "u": (game.wplayer.username, game.bplayer.username), "r": R2C[game.result], "d": game.date, "wr": game.wrating, "br": game.brating, "wb": game.wberserk, "bb": game.bberserk, } ) processed_games.add(game.id) await pairing_table.insert_many(pairing_documents) for user in self.leaderboard: await self.db_update_player(user, self.players[user]) if self.frequency == SHIELD: variant_name = self.variant + ("960" if self.chess960 else "") self.app["shield"][variant_name].append((winner, self.starts_at, self.id)) self.app["shield_owners"][variant_name] = winner def print_leaderboard(self): print("--- LEADERBOARD ---", self.id) for player, full_score in self.leaderboard.items()[:10]: print( "%20s %4s %30s %2s %s" % ( player.username, self.players[player].rating, self.players[player].points, full_score, self.players[player].performance, ) ) @property def create_discord_msg(self): tc = time_control_str(self.base, self.inc, self.byoyomi_period) tail960 = "960" if self.chess960 else "" return "%s: **%s%s** %s tournament starts at UTC %s, duration will be **%s** minutes" % ( self.created_by, self.variant, tail960, tc, self.starts_at.strftime("%Y.%m.%d %H:%M"), self.minutes, ) def notify_discord_msg(self, minutes): tc = time_control_str(self.base, self.inc, self.byoyomi_period) tail960 = "960" if self.chess960 else "" url = "https://www.pychess.org/tournament/%s" % self.id if minutes >= 60: time = int(minutes / 60) time_text = "hours" else: time = minutes time_text = "minutes" return "**%s%s** %s tournament starts in **%s** %s! %s" % ( self.variant, tail960, tc, time, time_text, url, )
def __init__( self, app, tournamentId, variant="chess", chess960=False, rated=True, before_start=5, minutes=45, name="", description="", fen="", base=1, inc=0, byoyomi_period=0, rounds=0, created_by="", created_at=None, starts_at=None, status=None, with_clock=True, frequency="", ): self.app = app self.id = tournamentId self.name = name self.description = description self.variant = variant self.rated = rated self.before_start = before_start # in minutes self.minutes = minutes # in minutes self.fen = fen self.base = base self.inc = inc self.byoyomi_period = byoyomi_period self.chess960 = chess960 self.rounds = rounds self.frequency = frequency self.created_by = created_by self.created_at = datetime.now(timezone.utc) if created_at is None else created_at if starts_at == "" or starts_at is None: self.starts_at = self.created_at + timedelta(seconds=int(before_start * 60)) else: self.starts_at = starts_at # TODO: calculate wave from TC, variant, number of players self.wave = timedelta(seconds=3) self.wave_delta = timedelta(seconds=1) self.current_round = 0 self.prev_pairing = None self.messages = collections.deque([], MAX_CHAT_LINES) self.spectators = set() self.players: dict[User, PlayerData] = {} self.leaderboard = ValueSortedDict(neg) self.leaderboard_keys_view = SortedKeysView(self.leaderboard) self.status = T_CREATED if status is None else status self.ongoing_games = 0 self.nb_players = 0 self.nb_games_finished = 0 self.w_win = 0 self.b_win = 0 self.draw = 0 self.nb_berserk = 0 self.nb_games_cached = -1 self.leaderboard_cache = {} self.first_pairing = False self.top_player = None self.top_game = None self.notify1 = False self.notify2 = False if minutes is None: self.ends_at = self.starts_at + timedelta(days=1) else: self.ends_at = self.starts_at + timedelta(minutes=minutes) if with_clock: self.clock_task = asyncio.create_task(self.clock())
async def init_state(app): # We have to put "kill" into a dict to prevent getting: # DeprecationWarning: Changing state of started or joined application is deprecated app["data"] = {"kill": False} if "db" not in app: app["db"] = None app["users"] = { "Random-Mover": User(app, bot=True, username="******"), "Fairy-Stockfish": User(app, bot=True, username="******"), "Discord-Relay": User(app, anon=True, username="******"), } app["users"]["Random-Mover"].bot_online = True app["websockets"] = {} app["seeks"] = {} app["games"] = {} app["chat"] = collections.deque([], 200) app["channels"] = set() app["highscore"] = {variant: ValueSortedDict(neg) for variant in VARIANTS} app["crosstable"] = {} # counters for games and users app["g_cnt"] = 0 app["u_cnt"] = 0 # last game played app["tv"] = None # fishnet active workers app["workers"] = set() # fishnet works app["works"] = {} # fishnet worker tasks app["fishnet"] = asyncio.PriorityQueue() # fishnet workers monitor app["fishnet_monitor"] = {} app["fishnet_versions"] = {} for key in FISHNET_KEYS: app["fishnet_monitor"][FISHNET_KEYS[key]] = collections.deque([], 50) rm = app["users"]["Random-Mover"] for variant in VARIANTS: variant960 = variant.endswith("960") variant_name = variant[:-3] if variant960 else variant byoyomi = variant == "janggi" or variant.endswith( "shogi") or variant == "shogun" seek = Seek(rm, variant_name, base=5, inc=3, level=0, chess960=variant960, byoyomi_period=1 if byoyomi else 0) app["seeks"][seek.id] = seek rm.seeks[seek.id] = seek ai = app["users"]["Fairy-Stockfish"] loop = asyncio.get_event_loop() loop.create_task(BOT_task(ai, app)) loop.create_task(BOT_task(rm, app)) # Configure templating. app["jinja"] = jinja2.Environment( loader=jinja2.FileSystemLoader("templates"), autoescape=jinja2.select_autoescape(["html"])) if app["db"] is None: return # Read users and highscore from db try: cursor = app["db"].user.find() async for doc in cursor: if doc["_id"] not in app["users"]: perfs = doc.get("perfs") if perfs is None: perfs = {variant: DEFAULT_PERF for variant in VARIANTS} app["users"][doc["_id"]] = User( app, username=doc["_id"], title=doc.get("title"), first_name=doc.get("first_name"), last_name=doc.get("last_name"), country=doc.get("country"), bot=doc.get("title") == "BOT", perfs=perfs, enabled=doc.get("enabled", True)) db_collections = await app["db"].list_collection_names() if "highscore" not in db_collections: await generate_highscore(app["db"]) cursor = app["db"].highscore.find() async for doc in cursor: app["highscore"][doc["_id"]] = ValueSortedDict(neg, doc["scores"]) if "crosstable" not in db_collections: await generate_crosstable(app["db"]) cursor = app["db"].crosstable.find() async for doc in cursor: app["crosstable"][doc["_id"]] = doc except Exception: print("Maybe mongodb is not running...") raise
def __init__(self, bin_count, history_length): self.history = ValueSortedDict() self.step = 0 self.thresholds = np.linspace(0, 1, bin_count - 1) self.history_length = history_length self.saturated = False
def test_copy(): temp = ValueSortedDict(identity, enumerate(reversed(alphabet))) that = temp.copy() assert temp == that assert temp._key != that._key
def test_init_kwargs(): temp = ValueSortedDict(None, a=0, b=1, c=2) assert len(temp) == 3 assert temp['a'] == 0 assert temp.iloc[0] == 'a' temp._check()
def pull_foreign_keys(self, dim: DimensionName) -> None: select_statement = self.foreign_key_lookups[dim] # type: Select from db import fetch self._foreign_keys[dim] = ValueSortedDict( {row[0]: str(row[1]) for row in fetch(select_statement)})
class SentenceRanker(object): """SentenceRanker manages a ranked list of sentences. Sentences are ranked based on a heuristic. Currently there is one heuristic named "concept density", but this can be expanded on in the future. sent_id = (doc_id, position) dict sentences_dict: Maps sent_id : ref to Sentence dict concept_to_sentences: Maps a concept to the sentences in which it occurs Concept : set([sent_ids]) ValueSortedDict ranks_to_sentences: [{sent_id : metric_value}] where list index corresponds to rank dict concept_weights: original concept weights k: Parameter that sets how many sentences are fed into ILP """ def __init__(self, sentences, concept_weights, summary_length, k, options): ''' :param sentences: List of Sentence objects :param concept_weights: Dictionary of weights for concept :param k: Number of k sentences that is fed into the ILP per feedback iteration ''' # Sets sentences_dict, concept_to_sent dict and ranked_to_sent dict if options['strategy'] == STRATIFIED: self.all_concept_weights = concept_weights else: self.all_concept_weights = deepcopy( concept_weights) # keep original concept weights self.initialize_sentences(sentences, concept_weights) self.k = k if options['relative_k']: self.k = int(k * self.get_corpus_size()) self.k_is_dynamic = options['dynamic_k'] self.summary_length = summary_length self.seen_sentences = set() self.important_concepts = set() # Interface for feedback self.get_input_sentences = self.get_top_k_sentences if options['strategy']: self.init_strategy(options) def initialize_sentences(self, sentences, concept_weights): '''Initializes sentences based on metric "concept density". creates: sentences_dict, concept_to_sentences, ranks_to_sentences ''' self.sentences_dict = {} self.concept_to_sentences = defaultdict(set) # Highest value --> first rank self.ranks_to_sentences = ValueSortedDict(lambda x: -x) for sent in sentences: # Create sentences_dict sent_id = (sent.doc_id, sent.position) self.sentences_dict[sent_id] = sent # Calculate concept density per sentence concept_density = 0 for concept in sent.concepts: concept_density += concept_weights[concept] # Create concept2sent dic self.concept_to_sentences[concept].add(sent_id) concept_density /= float(sent.length) # create ranks_to_sentences self.ranks_to_sentences[sent_id] = concept_density def update_ranking(self, new_accepts, new_rejects, new_implicits): ''' Changes top k sentences after feedback based on changed concept weights.''' # TODO: implement implicits changed_concepts = new_accepts + new_rejects for concept in changed_concepts: # Update affected sentences for sent_id in self.concept_to_sentences[concept]: sentence = self.sentences_dict[sent_id] concept_density = 0 for c in sentence.concepts: concept_density += self.all_concept_weights[c] concept_density /= float(sentence.length) # Update the metric and rank self.ranks_to_sentences[sent_id] = concept_density for concept in new_accepts: self.important_concepts.add(concept) if self.k_is_dynamic: self.set_k() self.k_history.append(self.k) return def update_weights(self, updated_weights): '''Update weights that have changed after weights have been recalculated.''' for key, value in updated_weights.items(): self.all_concept_weights[key] = value return def filter_concepts_of_top_k_sentences(self, new_accepts=[], new_rejects=[], k=None, sentences=None): ''' This method aggregates all relevant concepts based on top k sents and returns those. The ILP should only receive those concepts that are also in the subset of sentences which is passed to the ILP. For the intermediate summary, which is generated for oracle types 'feedback_ilp', and 'active_learning', the weights of the accepts and rejects should also be available in the returned dictionary (they might not be part of top k sentences anymore). ''' if sentences is None: sentences = self.get_top_k_sentences(k) concept_weights = {} for sent in sentences: for concept in sent.concepts: concept_weights[concept] = self.all_concept_weights[concept] if new_accepts + new_rejects: for concept in new_accepts + new_rejects: concept_weights[concept] = self.all_concept_weights[concept] return concept_weights def init_strategy(self, options): if options['strategy'] == BY_TIME: self.cost_model = CostModel('./algorithms/') self.cost_model.k_to_constraints = self.k_to_constraint_size self.determine_k = self.set_k_by_target_time elif options['strategy'] == BY_ENTROPY_INIT: self.k = self.set_k_by_entropy() self.determine_k = lambda: self.k elif options['strategy'] == BY_ENTROPY_ADAPT: self.determine_k = self.set_k_by_entropy elif options['strategy'] == BY_WINDOW: self.adaptive_window_size = options['adaptive_window_size'] self.determine_k = self.get_important_sentences elif options['strategy'] == BY_SWEEP: self.sweep_threshold = options['sweep_threshold'] self.get_input_sentences = self.get_distinct_top_k_sentences return elif options['strategy'] == BY_POS_LINK: self.get_input_sentences = self.get_sents_with_accepted_concepts if options['dynamic_k']: self.k_history = [] self.set_k() self.k_history.append(self.k) def set_k(self, k=None): if k is None and self.k_is_dynamic: chosen_k = self.determine_k() # Set k so there are enough concepts to fill L minimum_k = self.set_k_by_L() self.k = max(chosen_k, minimum_k) elif k is not None: self.k = k # Entropy Methods # def get_entropy(self, sentences): num_concepts = 0 concept_count = defaultdict(lambda: 0) for sent in sentences: for c in sent.concepts: num_concepts += 1 concept_count[c] += 1 S = num_concepts # Size of Sample self.num_of_concepts_in_summary = len(concept_count.keys()) base = len(self.all_concept_weights.keys()) entropy = 0.0 for concept, count in concept_count.items(): entropy -= (count / S) * log( (count / S), base) * self.all_concept_weights[concept] return entropy def set_k_by_entropy(self): k_to_entropy = [] top_k_sents = self.get_top_k_sentences(self.get_corpus_size()) num_concepts = 0 concept_count = defaultdict(lambda: 0) base = len(self.all_concept_weights.keys()) for k, sent in enumerate(top_k_sents, 1): for c in sent.concepts: num_concepts += 1 concept_count[c] += 1 entropy = 0.0 S = num_concepts for concept, count in concept_count.items(): entropy -= (count / S) * log( (count / S), base) * self.all_concept_weights[concept] k_to_entropy.append((k, entropy)) return max(k_to_entropy, key=lambda x: x[1])[0] def set_k_by_L(self): # TODO sequentially for k in range(1, self.get_corpus_size() + 1): bigram_count = len( set([ c for sent in self.get_top_k_sentences(k) for c in sent.concepts ])) # We count unique bigrams, hence / 2 if bigram_count / 2 >= self.summary_length: break return k def get_baseline_entropies(self): # Max, min baselines print(self.k_to_entropy) return [ m(self.k_to_entropy, key=lambda x: x[1])[0] for m in [max, min] ] def get_number_of_concepts(self): return self.num_of_concepts_in_summary # Time based strategy # def set_k_by_target_time(self): # TODO parametrize max_time max_time = 1 target_constraint_size = int(self.time_to_ilp_constraints(max_time)) occurences = 0 concepts = set() k = 0 while ((occurences + len(concepts) + 1) < target_constraint_size or len(concepts) < self.summary_length / 2): sent_id = self.ranks_to_sentences.iloc[k] k += 1 for c in set(self.sentences_dict[sent_id].concepts): occurences += 1 concepts.add(c) return k def k_to_constraint_size(self, k): if type(k) is pd.core.series.Series: s = {} se = [] unique_k = set(k) for i in unique_k: # s.append(self.get_constraint_size_for(i)) s[i] = self.get_constraint_size_for(i) for i in k: se.append(s[i]) return pd.Series(se) else: return self.get_constraint_size_for(int(k)) def time_to_ilp_constraints(self, t): return self.cost_model.constraints(t) def get_constraint_size_for(self, k): top_k_sent_ids = self.get_sentence_ids_for(k) occurences = 0 concepts = set() for s_id in top_k_sent_ids: for ci in set(self.sentences_dict[s_id].concepts): occurences += 1 concepts.add(ci) return occurences + len(concepts) + 1 # Adaptive Window # def get_important_sentences(self): num_of_important_sents = 0 # Get number of consecutive sentences that have at least one important concept for i, (sent_id, metric_value) in enumerate(self.ranks_to_sentences.items()): for concept in self.sentences_dict[sent_id].concepts: if concept in self.important_concepts: num_of_important_sents += 1 break if (i + 1) != num_of_important_sents: break return int(num_of_important_sents * (1 + self.adaptive_window_size)) # Redundancy Sweep # def get_distinct_top_k_sentences(self): top_k_sent_ids = list(self.ranks_to_sentences.keys()) distinct_sentences = [] seen_concepts = defaultdict(lambda: False) i = 0 while len(distinct_sentences) < self.k and i < self.get_corpus_size(): skip = False sent = self.sentences_dict[top_k_sent_ids[i]] counter = 0 for c in set(sent.concepts): if seen_concepts[c]: # +1 -> threshold = 0 means no overlap; 1 means one concept-overlap allowed if counter >= self.sweep_threshold + 1: skip = True break counter += 1 if not skip: distinct_sentences.append(sent) for c in sent.concepts: seen_concepts[c] = 1 i += 1 return distinct_sentences def get_sents_with_accepted_concepts(self): input_sentences = self.get_top_k_sentences() for sent_id in self.ranks_to_sentences.iloc[self.k:]: sent = self.sentences_dict[sent_id] for c in sent.concepts: if c in self.important_concepts: input_sentences.append(sent) break return input_sentences def get_top_k_sentences(self, k=None): print('### Top k %f' % self.k) if k is None: k = self.k # Statistic purposes: top_k_sent_ids = [ sent_id for sent_id in self.ranks_to_sentences.iloc[:k] ] self.seen_sentences |= set(top_k_sent_ids) return [ self.sentences_dict[sent_id] for sent_id in self.ranks_to_sentences.iloc[:k] ] def get_top_k_sentence_ids(self): return self.ranks_to_sentences.iloc[:self.k] def get_sentence_ids_for(self, k): return self.ranks_to_sentences.iloc[:k] def bisect_rank_by_value(self, value): '''workaround method as ValueSortedDict bisects by key (not sensible for sort order by value).''' self.ranks_to_sentences["bisect"] = value index = self.ranks_to_sentences.index("bisect") del self.ranks_to_sentences["bisect"] return index def get_corpus_size(self): return len(self.sentences_dict) # following: stuff for test purposes def rank_to_metric(self, rank): return self.ranks_to_sentences[self.ranks_to_sentences.iloc[rank]] def print_ranked_sentences(self, n=10, full_sentence=False): for rank, (doc_id, position) in enumerate(self.get_top_k_sentence_ids()[:n], 1): print("Rank: ", rank) if full_sentence: # self.sentences_dict[sent_id] returns the desired sentence print(self.sentences_dict[(doc_id, position)].untokenized_form) else: print("doc_id: {}, sentence_pos: {}".format(doc_id, position)) # self.ranks_to_sentences[sent_id] returns the metric print("Heuristic value: ", self.ranks_to_sentences[(doc_id, position)]) print("#-----#") def print_concept_map(self): i = 0 for key, value in self.concept_to_sentences.items(): print(key, ":", self.concept_to_sentences[key]) for entry in self.concept_to_sentences[key]: print(self.sentences_dict[entry].untokenized_form) i += 1 print("-------") if i >= 10: break
class Field(mp.Process): """Represents the detected playing field and it's contents. Also handles path creation.""" def __init__(self, id, car, gui): super(Field, self).__init__(target=self.dispatch, name='Field_{}_proc'.format(id)) self.exit = mp.Event() self.id = id self.car = car self.gui = gui # type: FieldGUI self.width = 150 self.height = 90 self.houses = dict() self.xes = ValueSortedDict() self.houseupdates = ValueSortedDict() # Essentially a priority queue with more functionality self.update_idx = 0 self.next_new_house = 0 self.potentials = np.zeros((H, W), dtype = float) self.que = mp.Queue(4) self.path = list() self.start() def stop(self): self.exit.set() def dispatch(self): while not self.exit.is_set(): try: func, args, kwargs = self.que.get(timeout=1) except Queue.Empty: pass else: type(self).__dict__[func](self, *args, **kwargs) def reset(self): """Invalidate static info.""" self.call('_reset') def _reset(self): self.houses.clear() self.houseupdates.clear() self.xes.clear() self.update_idx = 0 self.next_new_house = 0 def call(self, func, *args, **kwargs): self.que.put((func, args, kwargs)) # TODO: Change to put_nowait before the competition def update_houses(self, housearray): self.call('_update_houses', housearray) def _update_houses(self, housearray): self.update_idx += 1 for ha in housearray: # Get an iterator over all houses that have center x coordinate within 5 cm of the input value. # Then see if one of the houses is close enough to be considered the same. If there are, update it's # location point = ha[2:4] to_change = None for oldhouse_idx in self.xes.irange_key(point[0] - D, point[0] + D): # in cm. oldhouse = self.houses[oldhouse_idx] if -D < oldhouse.center.coords()[1] - point[1] < D and oldhouse.color == ha[1]: to_change = oldhouse_idx break if to_change is not None: house = self.houses[to_change] house.center.update(point) else: to_change = self.next_new_house self.next_new_house += 1 house = House(point, ha[1]) self.xes[to_change] = house.center.coords()[0] self.houses[to_change] = house self.houseupdates[to_change] = self.update_idx # if not self.update_idx % 5: self.clean_houses() self.update_potentials() if self.gui.enabled: self.draw_houses() self.draw_path(self._create_path((45, 75), self.car.yxintcoords())) def update_potentials(self): self.potentials = np.zeros((H, W), np.float32) for house in self.houses.values(): # type: House if house.color == House.RED: weight = R_WEIGHT elif house.color == House.GREEN: weight = G_WEIGHT else: weight = D_WEIGHT x, y = house.center.coords() self.potentials[int(y - 2 + 0.5):int(y + 2 + 0.5), int(x - 2 + 0.5):int(x + 2 + 0.5)] = weight cv2.GaussianBlur(self.potentials, (K_SIZE, K_SIZE), SIGMA, self.potentials, borderType=cv2.BORDER_CONSTANT) np.maximum(self.potentials, 0, self.potentials) def neighbours(self, point): """Return list of neighbour cells. 8 neighbours""" y, x = point ret = list() if y > 0: if x > 0: ret.append((y - 1, x - 1)) ret.append((y - 1, x)) if x < W - 1: ret.append((y - 1, x + 1)) if x > 0: ret.append((y, x - 1)) if x < W-1: ret.append((y, x + 1)) if y < H - 1: if x > 0: ret.append((y + 1, x - 1)) ret.append((y + 1, x)) if x < W - 1: ret.append((y + 1, x + 1)) return ret def create_path(self, start, end): self.call('_create_path', start, end) def _create_path(self, start, end): if H <= start[0] or W <= start[1] or \ H <= end[0] or W <= end[1] or \ start[0] < 0 or start[1] < 0 or \ end[0] < 0 or end[1] < 0: return [] from math import sqrt pq = SortedList(key=lambda x: -x[0]) # Priority Queue, with priority as the first element of entries parents = dict() parents[start] = None dist = dict() dist[start] = 0 # The cost function is straight-line distance + needed climb. def costfunc(point, other): stress = self.potentials[other] - self.potentials[point] if stress < 0: stress *= 0.8 pot = stress dist = 1. if point[0] == other[0] or point[1] == other[1] else 1.414 return pot + dist # The heuristic is straight-line distance plus needed climb def heuristic(point): pot = max(self.potentials[end] - self.potentials[point], 0) # pot = 0 # Might cause problems where it prefers to go around green houses when green houses have negative weight. dist = sqrt((end[0] - point[0]) ** 2 + (end[1] - point[1]) ** 2) return pot + dist pq.add((heuristic(start), start, None)) while True: try: est_u, u, p = pq.pop() parents[u] = p except IndexError: print('No path found.') u = None break if u == end: break for v in self.neighbours(u): new_dist = dist[u] + costfunc(u, v) if v not in dist or new_dist < dist[v]: dist[v] = new_dist est_v = new_dist + heuristic(v) pq.add((est_v, v, u)) path = [] while u is not None: path.append(u) u = parents[u] path.reverse() self.path = path return path def draw_path(self, path): if path: path = np.array(path) path = path.T img = np.frombuffer(self.gui.im_array.get_obj(), np.uint8).reshape((H, W, 3)) img[path[0], path[1], :] = [255, 255, 200] def draw_houses(self): frame = np.frombuffer(self.gui.im_array.get_obj(), np.uint8, H * W * 3).reshape((H, W, 3)) frame[:] = cv2.cvtColor(cv2.convertScaleAbs(self.potentials, alpha=1), cv2.COLOR_GRAY2BGR) for house in self.houses.values(): x, y = house.center.coords() xl, yl = max(x - 2, 0), max(y - 2, 0) xh, yh = min(x + 2, W - 1), min(y + 2, H - 1) if house.color == House.RED: col = (0, 0, 255) elif house.color == House.GREEN: col = (0, 255, 0) frame[yl:yh, xl:xh] = col if self.path: self.draw_path(self.path) def clean_houses(self): """Clean up houses that haven't been updated in 5 or more turns.""" indices = list(self.houseupdates.irange_key(None, self.update_idx - 5)) for idx in indices: del self.xes[idx] del self.houses[idx] del self.houseupdates[idx]