def get_eval_ratings(model_nums, eval_games):
# Map model_nums to a contigious range.
ordered = sorted(set(model_nums))
new_num = {}
for i, m in enumerate(ordered):
new_num[m] = i
def ilsr_data(eval_game):
p1, p2, black_won = eval_game
p1 = new_num[p1]
p2 = new_num[p2]
assert 0 <= p1 <= 3000
assert 0 <= p2 <= 3000
return (p1, p2) if black_won else (p2, p1)
pairs = list(map(ilsr_data, eval_games))
ilsr_param = choix.ilsr_pairwise(len(ordered),
pairs,
alpha=0.0001,
max_iter=200)
hessian = choix.opt.PairwiseFcts(pairs, penalty=.1).hessian(ilsr_param)
std_err = np.sqrt(np.diagonal(np.linalg.inv(hessian)))
# Elo conversion
elo_mult = 400 / math.log(10)
min_rating = min(ilsr_param)
ratings = {}
for model, param, err in zip(ordered, ilsr_param, std_err):
ratings[model] = (elo_mult * (param - min_rating), elo_mult * err)
return ratings
def infer_rankings(entries, f=None):
np.set_printoptions(precision=3, suppress=True)
parsed_entries = []
for line in entries:
values = line.split("\t")
if len(values) < 4:
print(f"match data incomplete {line}")
continue
a, aScore, b, bScore = values
parsed_entries.append((a, aScore, b, bScore))
if f:
parsed_entries = f(parsed_entries)
indexes = {}
matches = []
for a, aScore, b, bScore in parsed_entries:
try:
if a not in indexes:
indexes[a] = len(indexes)
if b not in indexes:
indexes[b] = len(indexes)
if int(aScore) > int(bScore):
matches.append((indexes[a], indexes[b]))
else:
matches.append((indexes[b], indexes[a]))
except:
print(f"match data incomplete {line}")
params = choix.ilsr_pairwise(len(indexes), matches, alpha=0.0001, max_iter=100)
return dict([(name, 1200 + params[indexes[name]] * 120) for name in sorted(indexes, key = lambda name: -params[indexes[name]])])
def get_ranking(comparisons):
comps = []
for pair, winner in comparisons.items():
pair_items = pair.split('_')
if pair_items[0] == 'check':
continue
other = pair_items[0] if pair_items[1] == winner else pair_items[1]
comps.append((conditions.index(winner), conditions.index(other)))
params = choix.ilsr_pairwise(len(conditions), comps, alpha=1e-5)
ranking = np.argsort(params)
return [conditions[r] for r in ranking][::-1]
def compute_ratings(model_ids, data=None):
""" Calculate ratings from win records
Args:
model_ids: dictionary of {(run, model_name): model_id}
data: list of tuples of (winner_id, loser_id)
Returns:
dictionary {(run, model_name): (rating, variance)}
"""
if data is None:
with sqlite3.connect("cbt_ratings.db") as db:
query = "select model_winner, model_loser from wins"
data = db.execute(query).fetchall()
data_ids = sorted(set(np.array(data).flatten()))
# Map data_ids to a contiguous range.
new_id = {}
for i, m in enumerate(data_ids):
new_id[m] = i
# Create inverse model_ids lookup
model_names = {v: k for k, v in model_ids.items()}
# A function to rewrite the data_ids in our pairs
def ilsr_data(d):
p1, p2 = d
p1 = new_id[p1]
p2 = new_id[p2]
return (p1, p2)
pairs = list(map(ilsr_data, data))
ilsr_param = choix.ilsr_pairwise(
len(data_ids),
pairs,
alpha=0.0001,
max_iter=800)
hessian = choix.opt.PairwiseFcts(pairs, penalty=.1).hessian(ilsr_param)
std_err = np.sqrt(np.diagonal(np.linalg.inv(hessian)))
# Elo conversion
elo_mult = 400 / math.log(10)
# Used to make all ratings positive.
min_rating = min(ilsr_param)
ratings = {}
for m_id, param, err in zip(data_ids, ilsr_param, std_err):
model_rating = (elo_mult * (param - min_rating), elo_mult * err)
ratings[model_names[m_id]] = model_rating
return ratings
def compute_ratings(data=None):
""" Returns the tuples of (model_id, rating, sigma)
N.B. that `model_id` here is NOT the model number in the run
'data' is tuples of (winner, loser) model_ids (not model numbers)
"""
if data is None:
with sqlite3.connect("ratings.db") as db:
data = db.execute(
"select model_winner, model_loser from wins").fetchall()
model_ids = set([d[0] for d in data]).union(set([d[1] for d in data]))
# Map model_ids to a contiguous range.
ordered = sorted(model_ids)
new_id = {}
for i, m in enumerate(ordered):
new_id[m] = i
# A function to rewrite the model_ids in our pairs
def ilsr_data(d):
p1, p2 = d
p1 = new_id[p1]
p2 = new_id[p2]
return (p1, p2)
pairs = list(map(ilsr_data, data))
ilsr_param = choix.ilsr_pairwise(len(ordered),
pairs,
alpha=0.0001,
max_iter=800)
hessian = choix.opt.PairwiseFcts(pairs, penalty=.1).hessian(ilsr_param)
std_err = np.sqrt(np.diagonal(np.linalg.inv(hessian)))
# Elo conversion
elo_mult = 400 / math.log(10)
min_rating = min(ilsr_param)
ratings = {}
for model_id, param, err in zip(ordered, ilsr_param, std_err):
ratings[model_id] = (elo_mult * (param - min_rating), elo_mult * err)
return ratings
def get_eval_ratings(model_nums, eval_games, priors):
# Map model_nums to a contigious range.
ordered = sorted(set(model_nums))
new_num = {}
for i, m in enumerate(ordered):
new_num[m] = i
# Transform priors (previous_ratings) (~elo~) back to Luce Spectral
# Elo conversion
elo_mult = 400 / math.log(10)
init = None
if priors:
avg = np.average(list(priors.values()))
init = [(priors.get(o, avg) - avg) / elo_mult for o in ordered]
def ilsr_data(eval_game):
p1, p2, black_won = eval_game
p1 = new_num[p1]
p2 = new_num[p2]
assert 0 <= p1 <= 10000, p1
assert 0 <= p2 <= 10000, p2
return (p1, p2) if black_won else (p2, p1)
pairs = list(map(ilsr_data, eval_games))
ilsr_param = choix.ilsr_pairwise(len(ordered),
pairs,
initial_params=init,
alpha=0.0001,
max_iter=1000)
hessian = choix.opt.PairwiseFcts(pairs, penalty=.1).hessian(ilsr_param)
std_err = np.sqrt(np.diagonal(np.linalg.inv(hessian)))
min_rating = min(ilsr_param)
ratings = {}
for model, param, err in zip(ordered, ilsr_param, std_err):
ratings[model] = (elo_mult * (param - min_rating), elo_mult * err)
return ratings
def compute_ratings(data, alpha=0.0001, max_iter=800):
""" Compute the ratings for a list of paired results.
'data' is tuples of (winner, loser) ids. IDs must be hashable, and no
games with the same players are permitted.
'alpha' & 'max_iter' are passed to choix.
Returns the tuples of (id, rating, sigma)
"""
if not data:
raise ValueError("No pairs for rating!")
# Get the unique list of models in the data.
data_ids = sorted(set(i for pair in data for i in pair))
# Map data_ids to a contiguous range.
new_id = {p: i for i, p in enumerate(data_ids)}
# Rewrite the model_ids in our pairs to the new ids
pairs = [tuple(map(new_id.get, pair)) for pair in data]
# Compute the ratings & something like error bars
ilsr_param = choix.ilsr_pairwise(
len(data_ids),
pairs,
alpha=alpha,
max_iter=max_iter)
hessian = choix.opt.PairwiseFcts(pairs, penalty=.1).hessian(ilsr_param)
std_err = np.sqrt(np.diagonal(np.linalg.inv(hessian)))
# "Elo" conversion
elo_mult = 400 / math.log(10)
# Make all ratings positive and zip them together.
min_rating = min(ilsr_param)
ratings = {}
for m_id, param, err in zip(data_ids, ilsr_param, std_err):
r = (elo_mult * (param - min_rating), elo_mult * err)
ratings[m_id] = r
return ratings
def pick_next_comparison(data, n_items):
params = choix.ilsr_pairwise(n_items, data, alpha=0.01)
count = 10
while count > 0:
x = random.randint(0, n_items - 1)
y = random.randint(0, n_items - 1)
while y == x:
y = random.randint(0, n_items - 1)
prob_x_wins, prob_y_wins = choix.probabilities([x, y], params)
if abs(
0.5 - prob_x_wins
) < 0.1: #if comparison has below 60% chance of being guessed correctly atm
return x, y
count -= 1
#if search 10 times for good comparison and fail pick random one
x = random.randint(0, n_items - 1)
y = random.randint(0, n_items - 1)
while y == x:
y = random.randint(0, n_items - 1)
return x, y
################# INPUT VALUES ######################
namps = amps.size #number of amplitudes
BTdata = []
for i in range(len(data)): #Go though each result file
for ii in range(np.size(data[i], axis=0)):
if data[i, ii, 3] == 1.0:
BTdata.append((np.int(
(data[i, ii, 2]) * 2), np.int((data[i, ii, 1]) * 2)))
else:
BTdata.append((np.int(
(data[i, ii, 1]) * 2), np.int((data[i, ii, 2]) * 2)))
BT = choix.ilsr_pairwise(namps, BTdata, 0.1)
pi = choix.probabilities(list(range(namps)), BT)
pyplot.close('all')
dx = 0.02
textX = 0.6 + dx
textY = 0.13
ax = pyplot.axes()
tsize = 12
pyplot.plot(Xdata, pi, 'o-', color='black')
pyplot.axvline(0.63, linestyle='--', color='gray')
ax.text(
textX,
textY,
(12, 2),
(12, 2),
(12, 2),
(4, 11),
(4, 11),
(11, 4),
(11, 4),
(11, 4),
# Final Stage
(9, 7),
(9, 7),
(9, 7),
(7, 9),
(7, 9),
(9, 12),
(9, 12),
(12, 9),
(12, 9),
(12, 9)
]
# Transfrom the data
for i in range(len(data)):
contest = list(data[i])
contest[0] -= 1
contest[1] -= 1
data[i] = tuple(contest)
# Do algorithm and get results
params = choix.ilsr_pairwise(n_items, data)
print(params)
print("ranking (worst to best):", np.argsort(params))
values = {}
for i in range(len(unique_vals_ranked)):
values[i] = unique_vals_ranked[i]
new_data = []
# this will convert something like [(7, 23)] into [(0, 1)] so that we can use choix
for comparison in data:
for key, value in values.items():
if value == comparison[0]:
new_x = key
elif value == comparison[1]:
new_y = key
new_data.append((new_x, new_y))
#---------------------------------- find order ------------------------------------------
params = choix.ilsr_pairwise(len(unique_vals_ranked), new_data, alpha=0.01)
order = np.argsort(params)
# if order is [3,12,7,9] we want to add all the other rows
# that have values in at the end so we can sort the list
for i in range(len(order)):
order[i] = values[order[i]]
for i in range(n_items):
if i not in order:
order = np.append(order, [i], axis=0)
#---------------------------------- rearrange list ------------------------------------------
x = df.reindex(order).reset_index(drop=True)
#--------------------------------------- allocate scores -------------------------------------
#buckets gives distribution that is downward sloping straight line (see excel file pairWiseComparisonDist)
f = open("../results.txt", "r")
pairs = dict()
for line in f:
if "::" in line:
linef = line.rstrip().rsplit("::")
pairs[linef[0]] = {
'id': linef[0],
'left': int(linef[1].rstrip().rsplit("-")[0].rsplit(".")[0]) - 1,
'right': int(linef[1].rstrip().rsplit("-")[1].rsplit(".")[0]) - 1
}
else:
pairs[line.rstrip()[:-2]]['choice'] = line.rstrip()[-1]
data = []
for k in pairs.keys():
if 'choice' in pairs[k]:
if pairs[k]['choice'] == 'A':
el = (pairs[k]['left'], pairs[k]['right'])
else:
el = (pairs[k]['right'], pairs[k]['left'])
data.append(el)
print("Nylon\tGemScore")
skills = choix.ilsr_pairwise(8, data, alpha=0.01)
print("=====================")
for i in range(len(skills)):
print(i + 1, "\t", skills[i])
import pandas as pd
import numpy as np
import choix
df = pd.read_csv("nba1617.csv")
teams = list(set(df.Home.unique()))
t = list(np.sort(teams))
data = []
for i in range(len(df.Visitor)):
if df.PTSV[i] > df.PTSH[i]:
el = (t.index(df.Visitor[i]), t.index(df.Home[i]))
else:
el = (t.index(df.Home[i]), t.index(df.Visitor[i]))
data.append(el)
skills = choix.ilsr_pairwise(30, data)
for i in range(30):
win_probs = []
for j in range(30):
win_probs.append(choix.probabilities([i, j], skills)[0])
print win_probs
if contest[1] == 'SFS': contest[1] = 0
if contest[1] == 'VAL': contest[1] = 1
if contest[1] == 'SHD': contest[1] = 2
if contest[1] == 'GLA': contest[1] = 3
if contest[1] == 'DAL': contest[1] = 4
if contest[1] == 'SEO': contest[1] = 5
if contest[1] == 'LDN': contest[1] = 6
if contest[1] == 'FLA': contest[1] = 7
if contest[1] == 'PHI': contest[1] = 8
if contest[1] == 'HOU': contest[1] = 9
if contest[1] == 'BOS': contest[1] = 10
if contest[1] == 'NYE': contest[1] = 11
competitions_without_draw[i] = tuple(contest)
# Do algorithm and get results
params = choix.ilsr_pairwise(n_items, competitions_without_draw)
#print(params)
leaderboard_BT = list(np.argsort(-params))
for i in range(len(leaderboard_BT)):
if leaderboard_BT[i] == 0: leaderboard_BT[i] = 'SFS'
if leaderboard_BT[i] == 1: leaderboard_BT[i] = 'VAL'
if leaderboard_BT[i] == 2: leaderboard_BT[i] = 'SHD'
if leaderboard_BT[i] == 3: leaderboard_BT[i] = 'GLA'
if leaderboard_BT[i] == 4: leaderboard_BT[i] = 'DAL'
if leaderboard_BT[i] == 5: leaderboard_BT[i] = 'SEO'
if leaderboard_BT[i] == 6: leaderboard_BT[i] = 'LDN'
if leaderboard_BT[i] == 7: leaderboard_BT[i] = 'FLA'
if leaderboard_BT[i] == 8: leaderboard_BT[i] = 'PHI'
if leaderboard_BT[i] == 9: leaderboard_BT[i] = 'HOU'
if leaderboard_BT[i] == 10: leaderboard_BT[i] = 'BOS'
if leaderboard_BT[i] == 11: leaderboard_BT[i] = 'NYE'