def sprt_finished(self):
"""Check whether SPRT test is finished."""
return stat_util.SPRT(
{
'wins': self.scores[0],
'losses': self.scores[1],
'draws': self.scores[2]
}, self.elo0, 0.05, self.elo1, 0.05, 200)["finished"]
def sync_update_task(self, run_id, task_id, stats, nps, spsa):
run = self.get_run(run_id)
if task_id >= len(run['tasks']):
return {'task_alive': False}
task = run['tasks'][task_id]
if not task['active'] or not task['pending']:
return {'task_alive': False}
# Guard against incorrect results
count_games = lambda d: d['wins'] + d['losses'] + d['draws']
num_games = count_games(stats)
old_num_games = count_games(
task['stats']) if 'stats' in task else num_games
spsa_games = count_games(spsa) if 'spsa' in run['args'] else 0
if num_games < old_num_games \
or (spsa_games > 0 and num_games <= 0) \
or (spsa_games > 0 and 'stats' in task and num_games <= old_num_games):
return {'task_alive': False}
flush = False
task['stats'] = stats
task['nps'] = nps
if num_games >= task['num_games']:
task['active'] = False
task['pending'] = False
flush = True
update_time = datetime.utcnow()
task['last_updated'] = update_time
run['last_updated'] = update_time
run['results_stale'] = True
# Update spsa results
if 'spsa' in run['args'] and spsa_games == spsa['num_games']:
self.update_spsa(run, spsa)
# Check if SPRT stopping is enabled
if 'sprt' in run['args']:
sprt = run['args']['sprt']
sprt_stats = stat_util.SPRT(self.get_results(run, False),
elo0=sprt['elo0'],
alpha=sprt['alpha'],
elo1=sprt['elo1'],
beta=sprt['beta'],
drawelo=sprt['drawelo'])
if sprt_stats['finished']:
run['args']['sprt']['state'] = sprt_stats['state']
self.stop_run(run_id, run)
flush = True
if (not 'spsa' in run['args'] or spsa_games == spsa['num_games']
or num_games >= task['num_games']
or len(spsa['w_params']) < 20):
self.buffer(run, flush)
return {'task_alive': task['active']}
def format_results(run_results, run):
result = {'style': '', 'info': []}
# win/loss/draw count
WLD = [run_results['wins'], run_results['losses'], run_results['draws']]
if 'spsa' in run['args']:
result['info'].append('%d/%d iterations' % (run['args']['spsa']['iter'], run['args']['spsa']['num_iter']))
result['info'].append('%d/%d games played' % (WLD[0] + WLD[1] + WLD[2], run['args']['num_games']))
return result
# If the score is 0% or 100% the formulas will crash
# anyway the statistics are only asymptotic
if WLD[0] == 0 or WLD[1] == 0:
result['info'].append('Pending...')
return result
state = 'unknown'
if 'sprt' in run['args']:
sprt = run['args']['sprt']
state = sprt.get('state', '')
stats = stat_util.SPRT(run_results,
elo0=sprt['elo0'],
alpha=sprt['alpha'],
elo1=sprt['elo1'],
beta=sprt['beta'],
drawelo=sprt['drawelo'])
result['llr'] = stats['llr']
result['info'].append('LLR: %.2f (%.2lf,%.2lf) [%.2f,%.2f]' % (stats['llr'], stats['lower_bound'], stats['upper_bound'], sprt['elo0'], sprt['elo1']))
else:
elo, elo95, los = stat_util.get_elo(WLD)
# Display the results
eloInfo = 'ELO: %.2f +-%.1f (95%%)' % (elo, elo95)
losInfo = 'LOS: %.1f%%' % (los * 100)
result['info'].append(eloInfo + ' ' + losInfo)
if los < 0.05:
state = 'rejected'
elif los > 0.95:
state = 'accepted'
result['info'].append('Total: %d W: %d L: %d D: %d' % (sum(WLD), WLD[0], WLD[1], WLD[2]))
if state == 'rejected':
if WLD[0] > WLD[1]:
result['style'] = 'yellow'
else:
result['style'] = '#FF6A6A'
elif state == 'accepted':
if 'sprt' in run['args'] and (float(sprt['elo0']) + float(sprt['elo1'])) < 0.0:
result['style'] = '#66CCFF'
else:
result['style'] = '#44EB44'
return result
def sprt_stats(scores, elo1, elo2):
s = stat_util.SPRT(
{
'wins': scores[0],
'losses': scores[1],
'draws': scores[2]
}, elo1, 0.05, elo2, 0.05, 200)
return "LLR: %.2f (%.2f,%.2f) [%.2f,%.2f]\n" % (
s['llr'], s['lower_bound'], s['upper_bound'], elo1, elo2)
def update_task(self, run_id, task_id, stats, nps, spsa):
run = self.get_run(run_id)
if task_id >= len(run['tasks']):
return {'task_alive': False}
task = run['tasks'][task_id]
if not task['active'] or not task['pending']:
return {'task_alive': False}
# Guard against incorrect results
num_games = stats['wins'] + stats['losses'] + stats['draws']
if 'stats' in task and num_games < task['stats']['wins'] + task[
'stats']['losses'] + task['stats']['draws']:
return {'task_alive': False}
task['stats'] = stats
task['nps'] = nps
if num_games >= task['num_games']:
task['active'] = False
task['pending'] = False
update_time = datetime.utcnow()
task['last_updated'] = update_time
run['last_updated'] = update_time
run['results_stale'] = True
# Update spsa results
if 'spsa' in run['args'] and spsa['wins'] + spsa['losses'] + spsa[
'draws'] == spsa['num_games']:
self.update_spsa(run, spsa)
self.runs.save(run)
# Check if SPRT stopping is enabled
if 'sprt' in run['args']:
sprt = run['args']['sprt']
sprt_stats = stat_util.SPRT(self.get_results(run),
elo0=sprt['elo0'],
alpha=sprt['alpha'],
elo1=sprt['elo1'],
beta=sprt['beta'],
drawelo=sprt['drawelo'])
if sprt_stats['finished']:
run['args']['sprt']['state'] = sprt_stats['state']
self.runs.save(run)
self.stop_run(run_id)
return {'task_alive': task['active']}
