def sprt_finished(self): """Check whether SPRT test is finished.""" return stat_util.SPRT( { 'wins': self.scores[0], 'losses': self.scores[1], 'draws': self.scores[2] }, self.elo0, 0.05, self.elo1, 0.05, 200)["finished"]
def sync_update_task(self, run_id, task_id, stats, nps, spsa): run = self.get_run(run_id) if task_id >= len(run['tasks']): return {'task_alive': False} task = run['tasks'][task_id] if not task['active'] or not task['pending']: return {'task_alive': False} # Guard against incorrect results count_games = lambda d: d['wins'] + d['losses'] + d['draws'] num_games = count_games(stats) old_num_games = count_games( task['stats']) if 'stats' in task else num_games spsa_games = count_games(spsa) if 'spsa' in run['args'] else 0 if num_games < old_num_games \ or (spsa_games > 0 and num_games <= 0) \ or (spsa_games > 0 and 'stats' in task and num_games <= old_num_games): return {'task_alive': False} flush = False task['stats'] = stats task['nps'] = nps if num_games >= task['num_games']: task['active'] = False task['pending'] = False flush = True update_time = datetime.utcnow() task['last_updated'] = update_time run['last_updated'] = update_time run['results_stale'] = True # Update spsa results if 'spsa' in run['args'] and spsa_games == spsa['num_games']: self.update_spsa(run, spsa) # Check if SPRT stopping is enabled if 'sprt' in run['args']: sprt = run['args']['sprt'] sprt_stats = stat_util.SPRT(self.get_results(run, False), elo0=sprt['elo0'], alpha=sprt['alpha'], elo1=sprt['elo1'], beta=sprt['beta'], drawelo=sprt['drawelo']) if sprt_stats['finished']: run['args']['sprt']['state'] = sprt_stats['state'] self.stop_run(run_id, run) flush = True if (not 'spsa' in run['args'] or spsa_games == spsa['num_games'] or num_games >= task['num_games'] or len(spsa['w_params']) < 20): self.buffer(run, flush) return {'task_alive': task['active']}
def format_results(run_results, run): result = {'style': '', 'info': []} # win/loss/draw count WLD = [run_results['wins'], run_results['losses'], run_results['draws']] if 'spsa' in run['args']: result['info'].append('%d/%d iterations' % (run['args']['spsa']['iter'], run['args']['spsa']['num_iter'])) result['info'].append('%d/%d games played' % (WLD[0] + WLD[1] + WLD[2], run['args']['num_games'])) return result # If the score is 0% or 100% the formulas will crash # anyway the statistics are only asymptotic if WLD[0] == 0 or WLD[1] == 0: result['info'].append('Pending...') return result state = 'unknown' if 'sprt' in run['args']: sprt = run['args']['sprt'] state = sprt.get('state', '') stats = stat_util.SPRT(run_results, elo0=sprt['elo0'], alpha=sprt['alpha'], elo1=sprt['elo1'], beta=sprt['beta'], drawelo=sprt['drawelo']) result['llr'] = stats['llr'] result['info'].append('LLR: %.2f (%.2lf,%.2lf) [%.2f,%.2f]' % (stats['llr'], stats['lower_bound'], stats['upper_bound'], sprt['elo0'], sprt['elo1'])) else: elo, elo95, los = stat_util.get_elo(WLD) # Display the results eloInfo = 'ELO: %.2f +-%.1f (95%%)' % (elo, elo95) losInfo = 'LOS: %.1f%%' % (los * 100) result['info'].append(eloInfo + ' ' + losInfo) if los < 0.05: state = 'rejected' elif los > 0.95: state = 'accepted' result['info'].append('Total: %d W: %d L: %d D: %d' % (sum(WLD), WLD[0], WLD[1], WLD[2])) if state == 'rejected': if WLD[0] > WLD[1]: result['style'] = 'yellow' else: result['style'] = '#FF6A6A' elif state == 'accepted': if 'sprt' in run['args'] and (float(sprt['elo0']) + float(sprt['elo1'])) < 0.0: result['style'] = '#66CCFF' else: result['style'] = '#44EB44' return result
def sprt_stats(scores, elo1, elo2): s = stat_util.SPRT( { 'wins': scores[0], 'losses': scores[1], 'draws': scores[2] }, elo1, 0.05, elo2, 0.05, 200) return "LLR: %.2f (%.2f,%.2f) [%.2f,%.2f]\n" % ( s['llr'], s['lower_bound'], s['upper_bound'], elo1, elo2)
def update_task(self, run_id, task_id, stats, nps, spsa): run = self.get_run(run_id) if task_id >= len(run['tasks']): return {'task_alive': False} task = run['tasks'][task_id] if not task['active'] or not task['pending']: return {'task_alive': False} # Guard against incorrect results num_games = stats['wins'] + stats['losses'] + stats['draws'] if 'stats' in task and num_games < task['stats']['wins'] + task[ 'stats']['losses'] + task['stats']['draws']: return {'task_alive': False} task['stats'] = stats task['nps'] = nps if num_games >= task['num_games']: task['active'] = False task['pending'] = False update_time = datetime.utcnow() task['last_updated'] = update_time run['last_updated'] = update_time run['results_stale'] = True # Update spsa results if 'spsa' in run['args'] and spsa['wins'] + spsa['losses'] + spsa[ 'draws'] == spsa['num_games']: self.update_spsa(run, spsa) self.runs.save(run) # Check if SPRT stopping is enabled if 'sprt' in run['args']: sprt = run['args']['sprt'] sprt_stats = stat_util.SPRT(self.get_results(run), elo0=sprt['elo0'], alpha=sprt['alpha'], elo1=sprt['elo1'], beta=sprt['beta'], drawelo=sprt['drawelo']) if sprt_stats['finished']: run['args']['sprt']['state'] = sprt_stats['state'] self.runs.save(run) self.stop_run(run_id) return {'task_alive': task['active']}