def reuse_example():
n, w = example('data/2021-02-06 12-16-42 wan-ticks.npz')
σs = []
contrast = 0
soln = activelo.solve(n, w)
for _ in range(20):
# Strip out this `soln=soln` to suppress soln reuse
soln = activelo.solve(n, w, soln=soln)
μ, Σ = soln.μ, soln.Σ
σ2 = np.diag(Σ) + Σ[contrast, contrast] - 2*Σ[contrast]
σs.append(σ2[-1]**.5)
σs = np.array(σs)
def play(self, agent):
size = self.worlds.boardsize
games = json.symmetric_games(f'mohex-{size}').pipe(append, 'agent')
wins = json.symmetric_wins(f'mohex-{size}').pipe(append, 'agent')
for result in self.history:
games.loc[result.names[0], result.names[1]] += result.games
games.loc[result.names[1], result.names[0]] += result.games
wins.loc[result.names[0], result.names[1]] += result.wins[0]
wins.loc[result.names[1], result.names[0]] += result.wins[1]
self.soln = activelo.solve(games, wins, soln=self.soln)
μ, σ = analysis.difference(self.soln, 'mohex-0.00', 'agent')
log.info(
f'Agent elo is {μ:.2f}±{σ:.2f} based on {int(games.loc["agent"].sum())} games'
)
stats.mean_std('elo-mohex', μ, σ)
imp = activelo.improvement(self.soln)
imp = pd.DataFrame(imp, games.index, games.index)
challenger = imp['agent'].idxmax()
randomness = float(challenger.split('-')[1])
self.mohex.random = randomness
results = common.evaluate(self.worlds, {
'agent': agent,
challenger: self.mohex
})
log.info(
f'Agent played {challenger}, {int(results[0].wins[0] + results[1].wins[1])}-{int(results[0].wins[1] + results[1].wins[0])}'
)
self.history.extend(results)
return arrdict.arrdict(games=games.loc['agent'].sum(), mean=μ, std=σ)
def play(self, agent):
self.soln = activelo.solve(self.games, self.wins, soln=self.soln)
μ, σ = analysis.difference(self.soln, 'mohex-0.00', 'agent')
log.info(
f'Agent elo is {μ:.2f}±{σ:.2f} based on {int(self.games.loc["agent"].sum())} games'
)
imp = activelo.improvement(self.soln)
imp = pd.DataFrame(imp, self.games.index, self.games.index)
challenger = imp['agent'].idxmax()
randomness = float(challenger.split('-')[1])
self.mohex.random = randomness
results = common.evaluate(self.worlds, {
'agent': agent,
challenger: self.mohex
})
log.info(
f'Agent played {challenger}, {int(results[0].wins[0] + results[1].wins[1])}-{int(results[0].wins[1] + results[1].wins[0])}'
)
for result in results:
self.games.loc[result.names[0], result.names[1]] += result.games
self.games.loc[result.names[1], result.names[0]] += result.games
self.wins.loc[result.names[0], result.names[1]] += result.wins[0]
self.wins.loc[result.names[1], result.names[0]] += result.wins[1]
return arrdict.arrdict(games=self.games.loc['agent'].sum(),
mean=μ,
std=σ), results
def saved_example(filename):
n, w = example(filename)
soln = activelo.solve(n, w)
activelo.plot(soln)
return soln
def generated_example():
N = 20
truth = torch.randn(N)
n = torch.randint(1, 50, (N, N))
d = truth[:, None] - truth[None, :]
w = torch.distributions.Binomial(n, 1/(1 + np.exp(-d))).sample()
trace = activelo.solve(n, w)
activelo.plot(trace)
def elos(run_name, names=None, queue=[]):
n = (json.symmetric_games(run_name).reindex(index=names,
columns=names).fillna(0))
w = (json.symmetric_wins(run_name).reindex(index=names,
columns=names).fillna(0))
for (i, j) in queue:
ni, nj = names[i], names[j]
w.loc[ni, nj] += (w.loc[ni, nj] + 1) / (n.loc[ni, nj] + 2)
n.loc[ni, nj] += 1
return activelo.solve(n.values, w.values)
def stds():
n, w = solvers.example('data/2021-02-06 12-16-42 wan-ticks.npz')
σs = []
contrast = 0
for _ in range(20):
soln = activelo.solve(n, w)
μ, Σ = soln.μ, soln.Σ
σ2 = np.diag(Σ) + Σ[contrast, contrast] - 2*Σ[contrast]
σs.append(σ2[-1]**.5)
σs = np.array(σs)
return σs
def elos(run, target=None, filter='.*'):
run = runs.resolve(run)
games, wins = json.symmetric(run)
games, wins = mask(games, wins, filter)
soln = activelo.solve(games.values, wins.values)
soln = pandas(soln, games.index)
if isinstance(target, int):
μ, σ = difference(soln, soln.μ.index[target])
elif isinstance(target, str):
μ, σ = difference(soln, target)
else:
μ, σ = soln.μ, pd.Series(np.diag(soln.Σ)**.5, games.index)
return pd.concat({'μ': μ, 'σ': σ}, 1)
def simulate(truth, n_games=256, σresid_tol=.1):
n_agents = len(truth)
wins = torch.zeros((n_agents, n_agents))
games = torch.zeros((n_agents, n_agents))
trace = []
ranks = torch.full((n_agents, ), 0.)
while True:
soln = activelo.solve(games, wins)
ranks = torch.as_tensor(soln.μ)
black, white = activelo.suggest(soln)
black_wins = torch.distributions.Binomial(
n_games, winrate(truth[black], truth[white])).sample()
wins[black, white] += black_wins
wins[white, black] += n_games - black_wins
games[black, white] += n_games
games[white, black] += n_games
soln['n'] = games.clone()
soln['w'] = wins.clone()
soln['σresid'] = residual_vs_mean(soln.Σ).mean()**.5
soln['resid_var'] = resid_var(ranks, truth)
trace.append(
arrdict.arrdict({k: v
for k, v in soln.items() if k != 'trace'}))
plt.close()
from IPython import display
display.clear_output(wait=True)
display.display(plot(trace, truth))
if soln.σresid < σresid_tol:
break
trace = arrdict.stack(trace)
return trace
def errors(run=-1, filter='.*'):
run = runs.resolve(run)
games, wins = database.symmetric(run)
games, wins = analysis.mask(games, wins, filter)
soln = activelo.solve(games.values, wins.values)
rates = wins / games
expected = 1 / (1 + np.exp(-soln.μ[:, None] + soln.μ[None, :]))
actual = rates.where(games > 0, np.nan).values
resid_var = np.nanmean((actual - expected)**2) / np.nanmean(actual**2)
corr = np.corrcoef(actual[~np.isnan(actual)],
expected[~np.isnan(actual)])[0, 1]
mohex = stats.pandas(
run, 'elo-mohex',
'μ').pipe(lambda df: df.ffill().where(df.bfill().notnull()))
mohex.index = (mohex.index - mohex.index[0]).total_seconds(
) / 900 #TODO: Generalise this to non-15-min snapshots
fig = plt.figure()
gs = plt.GridSpec(4, 3, fig, height_ratios=[20, 1, 20, 1])
fig.set_size_inches(18, 12)
# Top row
cmap = copy.copy(plt.cm.RdBu)
cmap.set_bad('lightgrey')
kwargs = dict(cmap=cmap, vmin=0, vmax=1, aspect=1)
ax = plt.subplot(gs[0, 0])
ax.imshow(actual, **kwargs)
ax.set_title('actual')
ax = plt.subplot(gs[0, 1])
im = ax.imshow(expected, **kwargs)
ax.set_title('expected')
ax = plt.subplot(gs[1, :2])
plt.colorbar(im, cax=ax, orientation='horizontal')
# Top right
ax = plt.subplot(gs[0, 2])
elos = analysis.elos(run, target=0)
ax.errorbar(np.arange(len(elos)),
elos.μ,
yerr=elos.σ,
marker='.',
capsize=2,
linestyle='')
ax.set_title('elos v. first')
ax.grid()
# Bottom left
ax = plt.subplot(gs[2, 0])
im = ax.imshow(actual - expected, vmin=-1, vmax=+1, aspect=1, cmap=cmap)
ax.set_title('error')
ax = plt.subplot(gs[3, 0])
plt.colorbar(im, cax=ax, orientation='horizontal')
# ax.annotate(f'resid var: {resid_var:.0%}, corr: {corr:.0%}', (.5, -1.2), ha='center', xycoords='axes fraction')
# Bottom middle
ax = plt.subplot(gs[2, 1])
se = (expected * (1 - expected) / games)**.5
im = ax.imshow((actual - expected) / se,
vmin=-3,
vmax=+3,
aspect=1,
cmap='RdBu')
ax.set_title('standard error')
ax = plt.subplot(gs[3, 1])
plt.colorbar(im, cax=ax, orientation='horizontal')
# ax.annotate(f'resid var: {resid_var:.0%}, corr: {corr:.0%}', (.5, -1.2), ha='center', xycoords='axes fraction')
# Bottom right
ax = plt.subplot(gs[2, 2])
im = mohex.plot(ax=ax, grid=True)
ax.set_title('elos v. mohex')
ax.set_xlabel('')
def plot_mohex(run):
import matplotlib.pyplot as plt
import copy
games, wins = database.symmetric(run)
games, wins = analysis.mask(games, wins, '.*')
soln = activelo.solve(games.values, wins.values)
rates = wins / games
expected = 1 / (1 + np.exp(-soln.μ[:, None] + soln.μ[None, :]))
actual = rates.where(games > 0, np.nan).values
fig = plt.figure()
gs = plt.GridSpec(4, 3, fig, height_ratios=[20, 1, 20, 1])
fig.set_size_inches(18, 12)
# Top row
cmap = copy.copy(plt.cm.RdBu)
cmap.set_bad('lightgrey')
kwargs = dict(cmap=cmap, vmin=0, vmax=1, aspect=1)
ax = plt.subplot(gs[0, 0])
ax.imshow(actual, **kwargs)
ax.set_title('actual')
ax = plt.subplot(gs[0, 1])
im = ax.imshow(expected, **kwargs)
ax.set_title('expected')
ax = plt.subplot(gs[1, :2])
plt.colorbar(im, cax=ax, orientation='horizontal')
# Top right
std = (soln.σd[0, :]**2).mean()**.5
ax = plt.subplot(gs[0, 2])
ax.errorbar(np.arange(len(soln.μ)),
soln.μd[0, :],
yerr=soln.σd[0, :],
marker='.',
capsize=2,
linestyle='')
ax.set_title(f'elos v. first, σ = {std:.2f}')
ax.grid()
# Bottom left
ax = plt.subplot(gs[2, 0])
im = ax.imshow(actual - expected, vmin=-1, vmax=+1, aspect=1, cmap=cmap)
ax.set_title('error')
ax = plt.subplot(gs[3, 0])
plt.colorbar(im, cax=ax, orientation='horizontal')
# ax.annotate(f'resid var: {resid_var:.0%}, corr: {corr:.0%}', (.5, -1.2), ha='center', xycoords='axes fraction')
# Bottom middle
ax = plt.subplot(gs[2, 1])
se = (expected * (1 - expected) / games)**.5
im = ax.imshow((actual - expected) / se,
vmin=-3,
vmax=+3,
aspect=1,
cmap='RdBu')
ax.set_title('standard error')
ax = plt.subplot(gs[3, 1])
plt.colorbar(im, cax=ax, orientation='horizontal')
# Bottom right
ax = plt.subplot(gs[2, 2])
im = ax.imshow(games, aspect=1, cmap='Reds')
ax.set_title('counts')
ax = plt.subplot(gs[3, 2])
plt.colorbar(im, cax=ax, orientation='horizontal')