def get_eval_ratings(model_nums, eval_games): # Map model_nums to a contigious range. ordered = sorted(set(model_nums)) new_num = {} for i, m in enumerate(ordered): new_num[m] = i def ilsr_data(eval_game): p1, p2, black_won = eval_game p1 = new_num[p1] p2 = new_num[p2] assert 0 <= p1 <= 3000 assert 0 <= p2 <= 3000 return (p1, p2) if black_won else (p2, p1) pairs = list(map(ilsr_data, eval_games)) ilsr_param = choix.ilsr_pairwise(len(ordered), pairs, alpha=0.0001, max_iter=200) hessian = choix.opt.PairwiseFcts(pairs, penalty=.1).hessian(ilsr_param) std_err = np.sqrt(np.diagonal(np.linalg.inv(hessian))) # Elo conversion elo_mult = 400 / math.log(10) min_rating = min(ilsr_param) ratings = {} for model, param, err in zip(ordered, ilsr_param, std_err): ratings[model] = (elo_mult * (param - min_rating), elo_mult * err) return ratings
def infer_rankings(entries, f=None): np.set_printoptions(precision=3, suppress=True) parsed_entries = [] for line in entries: values = line.split("\t") if len(values) < 4: print(f"match data incomplete {line}") continue a, aScore, b, bScore = values parsed_entries.append((a, aScore, b, bScore)) if f: parsed_entries = f(parsed_entries) indexes = {} matches = [] for a, aScore, b, bScore in parsed_entries: try: if a not in indexes: indexes[a] = len(indexes) if b not in indexes: indexes[b] = len(indexes) if int(aScore) > int(bScore): matches.append((indexes[a], indexes[b])) else: matches.append((indexes[b], indexes[a])) except: print(f"match data incomplete {line}") params = choix.ilsr_pairwise(len(indexes), matches, alpha=0.0001, max_iter=100) return dict([(name, 1200 + params[indexes[name]] * 120) for name in sorted(indexes, key = lambda name: -params[indexes[name]])])
def get_ranking(comparisons): comps = [] for pair, winner in comparisons.items(): pair_items = pair.split('_') if pair_items[0] == 'check': continue other = pair_items[0] if pair_items[1] == winner else pair_items[1] comps.append((conditions.index(winner), conditions.index(other))) params = choix.ilsr_pairwise(len(conditions), comps, alpha=1e-5) ranking = np.argsort(params) return [conditions[r] for r in ranking][::-1]
def compute_ratings(model_ids, data=None): """ Calculate ratings from win records Args: model_ids: dictionary of {(run, model_name): model_id} data: list of tuples of (winner_id, loser_id) Returns: dictionary {(run, model_name): (rating, variance)} """ if data is None: with sqlite3.connect("cbt_ratings.db") as db: query = "select model_winner, model_loser from wins" data = db.execute(query).fetchall() data_ids = sorted(set(np.array(data).flatten())) # Map data_ids to a contiguous range. new_id = {} for i, m in enumerate(data_ids): new_id[m] = i # Create inverse model_ids lookup model_names = {v: k for k, v in model_ids.items()} # A function to rewrite the data_ids in our pairs def ilsr_data(d): p1, p2 = d p1 = new_id[p1] p2 = new_id[p2] return (p1, p2) pairs = list(map(ilsr_data, data)) ilsr_param = choix.ilsr_pairwise( len(data_ids), pairs, alpha=0.0001, max_iter=800) hessian = choix.opt.PairwiseFcts(pairs, penalty=.1).hessian(ilsr_param) std_err = np.sqrt(np.diagonal(np.linalg.inv(hessian))) # Elo conversion elo_mult = 400 / math.log(10) # Used to make all ratings positive. min_rating = min(ilsr_param) ratings = {} for m_id, param, err in zip(data_ids, ilsr_param, std_err): model_rating = (elo_mult * (param - min_rating), elo_mult * err) ratings[model_names[m_id]] = model_rating return ratings
def compute_ratings(data=None): """ Returns the tuples of (model_id, rating, sigma) N.B. that `model_id` here is NOT the model number in the run 'data' is tuples of (winner, loser) model_ids (not model numbers) """ if data is None: with sqlite3.connect("ratings.db") as db: data = db.execute( "select model_winner, model_loser from wins").fetchall() model_ids = set([d[0] for d in data]).union(set([d[1] for d in data])) # Map model_ids to a contiguous range. ordered = sorted(model_ids) new_id = {} for i, m in enumerate(ordered): new_id[m] = i # A function to rewrite the model_ids in our pairs def ilsr_data(d): p1, p2 = d p1 = new_id[p1] p2 = new_id[p2] return (p1, p2) pairs = list(map(ilsr_data, data)) ilsr_param = choix.ilsr_pairwise(len(ordered), pairs, alpha=0.0001, max_iter=800) hessian = choix.opt.PairwiseFcts(pairs, penalty=.1).hessian(ilsr_param) std_err = np.sqrt(np.diagonal(np.linalg.inv(hessian))) # Elo conversion elo_mult = 400 / math.log(10) min_rating = min(ilsr_param) ratings = {} for model_id, param, err in zip(ordered, ilsr_param, std_err): ratings[model_id] = (elo_mult * (param - min_rating), elo_mult * err) return ratings
def get_eval_ratings(model_nums, eval_games, priors): # Map model_nums to a contigious range. ordered = sorted(set(model_nums)) new_num = {} for i, m in enumerate(ordered): new_num[m] = i # Transform priors (previous_ratings) (~elo~) back to Luce Spectral # Elo conversion elo_mult = 400 / math.log(10) init = None if priors: avg = np.average(list(priors.values())) init = [(priors.get(o, avg) - avg) / elo_mult for o in ordered] def ilsr_data(eval_game): p1, p2, black_won = eval_game p1 = new_num[p1] p2 = new_num[p2] assert 0 <= p1 <= 10000, p1 assert 0 <= p2 <= 10000, p2 return (p1, p2) if black_won else (p2, p1) pairs = list(map(ilsr_data, eval_games)) ilsr_param = choix.ilsr_pairwise(len(ordered), pairs, initial_params=init, alpha=0.0001, max_iter=1000) hessian = choix.opt.PairwiseFcts(pairs, penalty=.1).hessian(ilsr_param) std_err = np.sqrt(np.diagonal(np.linalg.inv(hessian))) min_rating = min(ilsr_param) ratings = {} for model, param, err in zip(ordered, ilsr_param, std_err): ratings[model] = (elo_mult * (param - min_rating), elo_mult * err) return ratings
def compute_ratings(data, alpha=0.0001, max_iter=800): """ Compute the ratings for a list of paired results. 'data' is tuples of (winner, loser) ids. IDs must be hashable, and no games with the same players are permitted. 'alpha' & 'max_iter' are passed to choix. Returns the tuples of (id, rating, sigma) """ if not data: raise ValueError("No pairs for rating!") # Get the unique list of models in the data. data_ids = sorted(set(i for pair in data for i in pair)) # Map data_ids to a contiguous range. new_id = {p: i for i, p in enumerate(data_ids)} # Rewrite the model_ids in our pairs to the new ids pairs = [tuple(map(new_id.get, pair)) for pair in data] # Compute the ratings & something like error bars ilsr_param = choix.ilsr_pairwise( len(data_ids), pairs, alpha=alpha, max_iter=max_iter) hessian = choix.opt.PairwiseFcts(pairs, penalty=.1).hessian(ilsr_param) std_err = np.sqrt(np.diagonal(np.linalg.inv(hessian))) # "Elo" conversion elo_mult = 400 / math.log(10) # Make all ratings positive and zip them together. min_rating = min(ilsr_param) ratings = {} for m_id, param, err in zip(data_ids, ilsr_param, std_err): r = (elo_mult * (param - min_rating), elo_mult * err) ratings[m_id] = r return ratings
def pick_next_comparison(data, n_items): params = choix.ilsr_pairwise(n_items, data, alpha=0.01) count = 10 while count > 0: x = random.randint(0, n_items - 1) y = random.randint(0, n_items - 1) while y == x: y = random.randint(0, n_items - 1) prob_x_wins, prob_y_wins = choix.probabilities([x, y], params) if abs( 0.5 - prob_x_wins ) < 0.1: #if comparison has below 60% chance of being guessed correctly atm return x, y count -= 1 #if search 10 times for good comparison and fail pick random one x = random.randint(0, n_items - 1) y = random.randint(0, n_items - 1) while y == x: y = random.randint(0, n_items - 1) return x, y
################# INPUT VALUES ###################### namps = amps.size #number of amplitudes BTdata = [] for i in range(len(data)): #Go though each result file for ii in range(np.size(data[i], axis=0)): if data[i, ii, 3] == 1.0: BTdata.append((np.int( (data[i, ii, 2]) * 2), np.int((data[i, ii, 1]) * 2))) else: BTdata.append((np.int( (data[i, ii, 1]) * 2), np.int((data[i, ii, 2]) * 2))) BT = choix.ilsr_pairwise(namps, BTdata, 0.1) pi = choix.probabilities(list(range(namps)), BT) pyplot.close('all') dx = 0.02 textX = 0.6 + dx textY = 0.13 ax = pyplot.axes() tsize = 12 pyplot.plot(Xdata, pi, 'o-', color='black') pyplot.axvline(0.63, linestyle='--', color='gray') ax.text( textX, textY,
(12, 2), (12, 2), (12, 2), (4, 11), (4, 11), (11, 4), (11, 4), (11, 4), # Final Stage (9, 7), (9, 7), (9, 7), (7, 9), (7, 9), (9, 12), (9, 12), (12, 9), (12, 9), (12, 9) ] # Transfrom the data for i in range(len(data)): contest = list(data[i]) contest[0] -= 1 contest[1] -= 1 data[i] = tuple(contest) # Do algorithm and get results params = choix.ilsr_pairwise(n_items, data) print(params) print("ranking (worst to best):", np.argsort(params))
values = {} for i in range(len(unique_vals_ranked)): values[i] = unique_vals_ranked[i] new_data = [] # this will convert something like [(7, 23)] into [(0, 1)] so that we can use choix for comparison in data: for key, value in values.items(): if value == comparison[0]: new_x = key elif value == comparison[1]: new_y = key new_data.append((new_x, new_y)) #---------------------------------- find order ------------------------------------------ params = choix.ilsr_pairwise(len(unique_vals_ranked), new_data, alpha=0.01) order = np.argsort(params) # if order is [3,12,7,9] we want to add all the other rows # that have values in at the end so we can sort the list for i in range(len(order)): order[i] = values[order[i]] for i in range(n_items): if i not in order: order = np.append(order, [i], axis=0) #---------------------------------- rearrange list ------------------------------------------ x = df.reindex(order).reset_index(drop=True) #--------------------------------------- allocate scores ------------------------------------- #buckets gives distribution that is downward sloping straight line (see excel file pairWiseComparisonDist)
f = open("../results.txt", "r") pairs = dict() for line in f: if "::" in line: linef = line.rstrip().rsplit("::") pairs[linef[0]] = { 'id': linef[0], 'left': int(linef[1].rstrip().rsplit("-")[0].rsplit(".")[0]) - 1, 'right': int(linef[1].rstrip().rsplit("-")[1].rsplit(".")[0]) - 1 } else: pairs[line.rstrip()[:-2]]['choice'] = line.rstrip()[-1] data = [] for k in pairs.keys(): if 'choice' in pairs[k]: if pairs[k]['choice'] == 'A': el = (pairs[k]['left'], pairs[k]['right']) else: el = (pairs[k]['right'], pairs[k]['left']) data.append(el) print("Nylon\tGemScore") skills = choix.ilsr_pairwise(8, data, alpha=0.01) print("=====================") for i in range(len(skills)): print(i + 1, "\t", skills[i])
import pandas as pd import numpy as np import choix df = pd.read_csv("nba1617.csv") teams = list(set(df.Home.unique())) t = list(np.sort(teams)) data = [] for i in range(len(df.Visitor)): if df.PTSV[i] > df.PTSH[i]: el = (t.index(df.Visitor[i]), t.index(df.Home[i])) else: el = (t.index(df.Home[i]), t.index(df.Visitor[i])) data.append(el) skills = choix.ilsr_pairwise(30, data) for i in range(30): win_probs = [] for j in range(30): win_probs.append(choix.probabilities([i, j], skills)[0]) print win_probs
if contest[1] == 'SFS': contest[1] = 0 if contest[1] == 'VAL': contest[1] = 1 if contest[1] == 'SHD': contest[1] = 2 if contest[1] == 'GLA': contest[1] = 3 if contest[1] == 'DAL': contest[1] = 4 if contest[1] == 'SEO': contest[1] = 5 if contest[1] == 'LDN': contest[1] = 6 if contest[1] == 'FLA': contest[1] = 7 if contest[1] == 'PHI': contest[1] = 8 if contest[1] == 'HOU': contest[1] = 9 if contest[1] == 'BOS': contest[1] = 10 if contest[1] == 'NYE': contest[1] = 11 competitions_without_draw[i] = tuple(contest) # Do algorithm and get results params = choix.ilsr_pairwise(n_items, competitions_without_draw) #print(params) leaderboard_BT = list(np.argsort(-params)) for i in range(len(leaderboard_BT)): if leaderboard_BT[i] == 0: leaderboard_BT[i] = 'SFS' if leaderboard_BT[i] == 1: leaderboard_BT[i] = 'VAL' if leaderboard_BT[i] == 2: leaderboard_BT[i] = 'SHD' if leaderboard_BT[i] == 3: leaderboard_BT[i] = 'GLA' if leaderboard_BT[i] == 4: leaderboard_BT[i] = 'DAL' if leaderboard_BT[i] == 5: leaderboard_BT[i] = 'SEO' if leaderboard_BT[i] == 6: leaderboard_BT[i] = 'LDN' if leaderboard_BT[i] == 7: leaderboard_BT[i] = 'FLA' if leaderboard_BT[i] == 8: leaderboard_BT[i] = 'PHI' if leaderboard_BT[i] == 9: leaderboard_BT[i] = 'HOU' if leaderboard_BT[i] == 10: leaderboard_BT[i] = 'BOS' if leaderboard_BT[i] == 11: leaderboard_BT[i] = 'NYE'