def stop_run(self, run_id): """Stops a run and runs auto-purge if it was enabled - Used by the website and API for manually stopping runs - Called during /api/update_task: - for stopping SPRT runs if the test is accepted or rejected - for stopping a run after all games are finished """ self.clear_params(run_id) # spsa stuff run = self.get_run(run_id) for task in run["tasks"]: task["active"] = False run["results_stale"] = True results = self.get_results(run, True) run["results_info"] = format_results(results, run) # De-couple the styling of the run from its finished status if run["results_info"]["style"] == "#44EB44": run["is_green"] = True elif run["results_info"]["style"] == "yellow": run["is_yellow"] = True run["finished"] = True self.buffer(run, True) # Publish the results of the run to the Fishcooking forum post_in_fishcooking_results(run) self.task_time = 0 # triggers a reload of self.task_runs # Auto-purge runs here. This may revive the run. if run["args"].get("auto_purge", True) and "spsa" not in run["args"]: message = self.purge_run(run) if message == "": print("Run {} was auto-purged".format(str(run_id)), flush=True) else: print( "Run {} was not auto-purged. Message: {}.".format( str(run_id), message), flush=True, )
def aggregate_unfinished_runs(self, username=None): unfinished_runs = self.get_unfinished_runs(username) runs = {"pending": [], "active": []} for run in unfinished_runs: state = ( "active" if any(task["active"] for task in run["tasks"]) else "pending" ) if state == "pending": run["cores"] = 0 runs[state].append(run) runs["pending"].sort( key=lambda run: ( run["args"]["priority"], run["args"]["itp"] if "itp" in run["args"] else 100, ) ) runs["active"].sort( reverse=True, key=lambda run: ( "sprt" in run["args"], run["args"].get("sprt", {}).get("llr", 0), "spsa" not in run["args"], run["results"]["wins"] + run["results"]["draws"] + run["results"]["losses"], ), ) # Calculate but don't save results_info on runs using info on current machines cores = 0 nps = 0 for m in self.get_machines(): concurrency = int(m["concurrency"]) cores += concurrency nps += concurrency * m["nps"] pending_hours = 0 for run in runs["pending"] + runs["active"]: if cores > 0: eta = remaining_hours(run) / cores pending_hours += eta results = self.get_results(run, False) run["results_info"] = format_results(results, run) if "Pending..." in run["results_info"]["info"]: if cores > 0: run["results_info"]["info"][0] += " (%.1f hrs)" % (eta) if "sprt" in run["args"]: sprt = run["args"]["sprt"] elo_model = sprt.get("elo_model", "BayesElo") if elo_model == "BayesElo": run["results_info"]["info"].append( ("[%.2f,%.2f]") % (sprt["elo0"], sprt["elo1"]) ) else: run["results_info"]["info"].append( ("{%.2f,%.2f}") % (sprt["elo0"], sprt["elo1"]) ) return (runs, pending_hours, cores, nps)
def stop_run(self, run_id, run=None): """ Stops a run and runs auto-purge if it was enabled - Used by the website and API for manually stopping runs - Called during /api/update_task: - for stopping SPRT runs if the test is accepted or rejected - for stopping a run after all games are finished """ self.clear_params(run_id) save_it = False if run is None: run = self.get_run(run_id) save_it = True run.pop('cores', None) run['tasks'] = [task for task in run['tasks'] if 'stats' in task] for task in run['tasks']: task['pending'] = False task['active'] = False if save_it: self.buffer(run, True) self.task_time = 0 # Auto-purge runs here purged = False if run['args'].get('auto_purge', True) and 'spsa' not in run['args']: if self.purge_run(run): purged = True run = self.get_run(run['_id']) results = self.get_results(run, True) run['results_info'] = format_results(results, run) self.buffer(run, True) if not purged: # The run is now finished and will no longer be updated after this run['finished'] = True results = self.get_results(run, True) run['results_info'] = format_results(results, run) # De-couple the styling of the run from its finished status if run['results_info']['style'] == '#44EB44': run['is_green'] = True elif run['results_info']['style'] == 'yellow': run['is_yellow'] = True self.buffer(run, True) # Publish the results of the run to the Fishcooking forum post_in_fishcooking_results(run)
def stop_run(self, run_id, run=None): """Stops a run and runs auto-purge if it was enabled - Used by the website and API for manually stopping runs - Called during /api/update_task: - for stopping SPRT runs if the test is accepted or rejected - for stopping a run after all games are finished """ self.clear_params(run_id) save_it = False if run is None: run = self.get_run(run_id) save_it = True run["tasks"] = [task for task in run["tasks"] if "stats" in task] for task in run["tasks"]: task["pending"] = False task["active"] = False if save_it: self.buffer(run, True) self.task_time = 0 # Auto-purge runs here purged = False if run["args"].get("auto_purge", True) and "spsa" not in run["args"]: if self.purge_run(run): purged = True run = self.get_run(run["_id"]) results = self.get_results(run, True) run["results_info"] = format_results(results, run) self.buffer(run, True) if not purged: # The run is now finished and will no longer be updated after this run["finished"] = True results = self.get_results(run, True) run["results_info"] = format_results(results, run) # De-couple the styling of the run from its finished status if run["results_info"]["style"] == "#44EB44": run["is_green"] = True elif run["results_info"]["style"] == "yellow": run["is_yellow"] = True self.buffer(run, True) # Publish the results of the run to the Fishcooking forum post_in_fishcooking_results(run)
def aggregate_unfinished_runs(self, username=None): unfinished_runs = self.get_unfinished_runs(username) runs = {'pending': [], 'active': []} for run in unfinished_runs: state = 'active' if any(task['active'] for task in run['tasks']) else 'pending' runs[state].append(run) runs['pending'].sort(key=lambda run: (run['args']['priority'], run['args']['itp'] if 'itp' in run['args'] else 100)) runs['active'].sort(reverse=True, key=lambda run: ( 'sprt' in run['args'], run['args'].get('sprt',{}).get('llr',0), 'spsa' not in run['args'], run['results']['wins'] + run['results']['draws'] + run['results']['losses'])) # Calculate but don't save results_info on runs using info on current machines cores = 0 nps = 0 for m in self.get_machines(): concurrency = int(m['concurrency']) cores += concurrency nps += concurrency * m['nps'] pending_hours = 0 for run in runs['pending'] + runs['active']: if cores > 0: eta = remaining_hours(run) / cores pending_hours += eta results = self.get_results(run, False) run['results_info'] = format_results(results, run) if 'Pending...' in run['results_info']['info']: if cores > 0: run['results_info']['info'][0] += ' (%.1f hrs)' % (eta) if 'sprt' in run['args']: sprt = run['args']['sprt'] elo_model = sprt.get('elo_model', 'BayesElo') if elo_model == 'BayesElo': run['results_info']['info'].append(('[%.2f,%.2f]') % (sprt['elo0'], sprt['elo1'])) else: run['results_info']['info'].append(('{%.2f,%.2f}') % (sprt['elo0'], sprt['elo1'])) return (runs, pending_hours, cores, nps)
def get_paginated_finished_runs(request): username = request.matchdict.get('username', '') success_only = request.params.get('success_only', False) yellow_only = request.params.get('yellow_only', False) ltc_only = request.params.get('ltc_only', False) page_idx = max(0, int(request.params.get('page', 1)) - 1) page_size = 50 finished_runs, num_finished_runs = request.rundb.get_finished_runs( username=username, success_only=success_only, yellow_only=yellow_only, ltc_only=ltc_only, skip=page_idx * page_size, limit=page_size) pages = [{ 'idx': 'Prev', 'url': '?page={}'.format(page_idx), 'state': 'disabled' if page_idx == 0 else '' }] for idx, _ in enumerate(range(0, num_finished_runs, page_size)): if idx < 5 or abs(page_idx - idx) < 5 or idx > (num_finished_runs / page_size) - 5: pages.append({ 'idx': idx + 1, 'url': '?page={}'.format(idx + 1), 'state': 'active' if page_idx == idx else '' }) elif pages[-1]['idx'] != '...': pages.append({'idx': '...', 'url': '', 'state': 'disabled'}) pages.append({ 'idx': 'Next', 'url': '?page={}'.format(page_idx + 2), 'state': 'disabled' if page_idx + 1 == len(pages) - 1 else '' }) for page in pages: if success_only: page['url'] += '&success_only=1' if yellow_only: page['url'] += '&yellow_only=1' if ltc_only: page['url'] += '<c_only=1' failed_runs = [] for run in finished_runs: # Ensure finished runs have results_info results = request.rundb.get_results(run) if 'results_info' not in run: run['results_info'] = format_results(results, run) # Look for failed runs if results['wins'] + results['losses'] + results['draws'] == 0: failed_runs.append(run) return { 'finished_runs': finished_runs, 'finished_runs_pages': pages, 'num_finished_runs': num_finished_runs, 'failed_runs': failed_runs, 'page_idx': page_idx, }
def tests_view(request): run = request.rundb.get_run(request.matchdict['id']) if run is None: raise exception_response(404) results = request.rundb.get_results(run) run['results_info'] = format_results(results, run) run_args = [('id', str(run['_id']), '')] for name in [ 'new_tag', 'new_signature', 'new_options', 'resolved_new', 'base_tag', 'base_signature', 'base_options', 'resolved_base', 'sprt', 'num_games', 'spsa', 'tc', 'threads', 'book', 'book_depth', 'auto_purge', 'priority', 'itp', 'username', 'tests_repo', 'info' ]: if name not in run['args']: continue value = run['args'][name] url = '' if name == 'new_tag' and 'msg_new' in run['args']: value += ' (' + run['args']['msg_new'][:50] + ')' if name == 'base_tag' and 'msg_base' in run['args']: value += ' (' + run['args']['msg_base'][:50] + ')' if name == 'sprt' and value != '-': value = 'elo0: %.2f alpha: %.2f elo1: %.2f beta: %.2f state: %s (%s)' % \ (value['elo0'], value['alpha'], value['elo1'], value['beta'], value.get('state', '-'), value.get('elo_model', 'BayesElo')) if name == 'spsa' and value != '-': iter_local = value['iter'] + 1 # assume at least one completed, # and avoid division by zero A = value['A'] alpha = value['alpha'] gamma = value['gamma'] summary = 'Iter: %d, A: %d, alpha %0.3f, gamma %0.3f, clipping %s, rounding %s' \ % (iter_local, A, alpha, gamma, value['clipping'] if 'clipping' in value else 'old', value['rounding'] if 'rounding' in value else 'deterministic') params = value['params'] value = [summary] for p in params: value.append([ p['name'], '{:.2f}'.format(p['theta']), int(p['start']), int(p['min']), int(p['max']), '{:.3f}'.format(p['c'] / (iter_local**gamma)), '{:.3f}'.format(p['a'] / (A + iter_local)**alpha) ]) if 'tests_repo' in run['args']: if name == 'new_tag': url = run['args']['tests_repo'] + '/commit/' + run['args'][ 'resolved_new'] elif name == 'base_tag': url = run['args']['tests_repo'] + '/commit/' + run['args'][ 'resolved_base'] elif name == 'tests_repo': url = value if name == 'spsa': run_args.append(('spsa', value, '')) else: try: strval = str(value) except: strval = value.encode('ascii', 'replace') strval = html.escape(strval) run_args.append((name, strval, url)) active = 0 cores = 0 for task in run['tasks']: if task['active']: active += 1 cores += task['worker_info']['concurrency'] last_updated = task.get('last_updated', datetime.datetime.min) task['last_updated'] = last_updated if run['args'].get('sprt'): page_title = 'SPRT {} vs {}'.format(run['args']['new_tag'], run['args']['base_tag']) elif run['args'].get('spsa'): page_title = 'SPSA {}'.format(run['args']['new_tag']) else: page_title = '{} games - {} vs {}'.format(run['args']['num_games'], run['args']['new_tag'], run['args']['base_tag']) return { 'run': run, 'run_args': run_args, 'page_title': page_title, 'approver': has_permission('approve_run', request.context, request), 'chi2': calculate_residuals(run), 'totals': '(%s active worker%s with %s core%s)' % (active, ('s' if active != 1 else ''), cores, ('s' if cores != 1 else '')) }
def tests_view(request): run = request.rundb.get_run(request.matchdict["id"]) if run is None: raise exception_response(404) results = request.rundb.get_results(run) run["results_info"] = format_results(results, run) run_args = [("id", str(run["_id"]), "")] if run.get("rescheduled_from"): run_args.append(("rescheduled_from", run["rescheduled_from"], "")) for name in [ "new_tag", "new_signature", "new_options", "resolved_new", "new_net", "base_tag", "base_signature", "base_options", "resolved_base", "base_net", "sprt", "num_games", "spsa", "tc", "threads", "book", "book_depth", "auto_purge", "priority", "itp", "username", "tests_repo", "info", ]: if name not in run["args"]: continue value = run["args"][name] url = "" if name == "new_tag" and "msg_new" in run["args"]: value += " (" + run["args"]["msg_new"][:50] + ")" if name == "base_tag" and "msg_base" in run["args"]: value += " (" + run["args"]["msg_base"][:50] + ")" if name == "sprt" and value != "-": value = "elo0: %.2f alpha: %.2f elo1: %.2f beta: %.2f state: %s (%s)" % ( value["elo0"], value["alpha"], value["elo1"], value["beta"], value.get("state", "-"), value.get("elo_model", "BayesElo"), ) if name == "spsa" and value != "-": iter_local = value["iter"] + 1 # assume at least one completed, # and avoid division by zero A = value["A"] alpha = value["alpha"] gamma = value["gamma"] summary = ( "Iter: %d, A: %d, alpha %0.3f, gamma %0.3f, clipping %s, rounding %s" % ( iter_local, A, alpha, gamma, value["clipping"] if "clipping" in value else "old", value["rounding"] if "rounding" in value else "deterministic", )) params = value["params"] value = [summary] for p in params: value.append([ p["name"], "{:.2f}".format(p["theta"]), int(p["start"]), int(p["min"]), int(p["max"]), "{:.3f}".format(p["c"] / (iter_local**gamma)), "{:.3f}".format(p["a"] / (A + iter_local)**alpha), ]) if "tests_repo" in run["args"]: if name == "new_tag": url = (run["args"]["tests_repo"] + "/commit/" + run["args"]["resolved_new"]) elif name == "base_tag": url = (run["args"]["tests_repo"] + "/commit/" + run["args"]["resolved_base"]) elif name == "tests_repo": url = value if name == "spsa": run_args.append(("spsa", value, "")) else: try: strval = str(value) except: strval = value.encode("ascii", "replace") if name not in ["new_tag", "base_tag"]: strval = html.escape(strval) run_args.append((name, strval, url)) active = 0 cores = 0 for task in run["tasks"]: if task["active"]: active += 1 cores += task["worker_info"]["concurrency"] last_updated = task.get("last_updated", datetime.datetime.min) task["last_updated"] = last_updated if run["args"].get("sprt"): page_title = "SPRT {} vs {}".format(run["args"]["new_tag"], run["args"]["base_tag"]) elif run["args"].get("spsa"): page_title = "SPSA {}".format(run["args"]["new_tag"]) else: page_title = "{} games - {} vs {}".format(run["args"]["num_games"], run["args"]["new_tag"], run["args"]["base_tag"]) return { "run": run, "run_args": run_args, "page_title": page_title, "approver": has_permission("approve_run", request.context, request), "chi2": calculate_residuals(run), "totals": "(%s active worker%s with %s core%s)" % (active, ("s" if active != 1 else ""), cores, ("s" if cores != 1 else "")), }
def get_paginated_finished_runs(request): username = request.matchdict.get("username", "") success_only = request.params.get("success_only", False) yellow_only = request.params.get("yellow_only", False) ltc_only = request.params.get("ltc_only", False) page_idx = max(0, int(request.params.get("page", 1)) - 1) page_size = 25 finished_runs, num_finished_runs = request.rundb.get_finished_runs( username=username, success_only=success_only, yellow_only=yellow_only, ltc_only=ltc_only, skip=page_idx * page_size, limit=page_size, ) pages = [{ "idx": "Prev", "url": "?page={}".format(page_idx), "state": "disabled" if page_idx == 0 else "", }] for idx, _ in enumerate(range(0, num_finished_runs, page_size)): if (idx < 5 or abs(page_idx - idx) < 5 or idx > (num_finished_runs / page_size) - 5): pages.append({ "idx": idx + 1, "url": "?page={}".format(idx + 1), "state": "active" if page_idx == idx else "", }) elif pages[-1]["idx"] != "...": pages.append({"idx": "...", "url": "", "state": "disabled"}) pages.append({ "idx": "Next", "url": "?page={}".format(page_idx + 2), "state": "disabled" if page_idx + 1 == len(pages) - 1 else "", }) for page in pages: if success_only: page["url"] += "&success_only=1" if yellow_only: page["url"] += "&yellow_only=1" if ltc_only: page["url"] += "<c_only=1" failed_runs = [] for run in finished_runs: # Ensure finished runs have results_info results = request.rundb.get_results(run) if "results_info" not in run: run["results_info"] = format_results(results, run) # Look for failed runs if "failed" in run: failed_runs.append(run) return { "finished_runs": finished_runs, "finished_runs_pages": pages, "num_finished_runs": num_finished_runs, "failed_runs": failed_runs, "page_idx": page_idx, }
def tests_view(request): run = request.rundb.get_run(request.matchdict["id"]) if run is None: raise exception_response(404) results = request.rundb.get_results(run) run["results_info"] = format_results(results, run) run_args = [("id", str(run["_id"]), "")] if run.get("rescheduled_from"): run_args.append(("rescheduled_from", run["rescheduled_from"], "")) for name in [ "new_tag", "new_signature", "new_options", "resolved_new", "new_net", "base_tag", "base_signature", "base_options", "resolved_base", "base_net", "sprt", "num_games", "spsa", "tc", "new_tc", "threads", "book", "book_depth", "auto_purge", "priority", "itp", "username", "tests_repo", "adjudication", "info", ]: if name not in run["args"]: continue value = run["args"][name] url = "" if name == "new_tag" and "msg_new" in run["args"]: value += " (" + run["args"]["msg_new"][:50] + ")" if name == "base_tag" and "msg_base" in run["args"]: value += " (" + run["args"]["msg_base"][:50] + ")" if name == "sprt" and value != "-": value = "elo0: {:.2f} alpha: {:.2f} elo1: {:.2f} beta: {:.2f} state: {} ({})".format( value["elo0"], value["alpha"], value["elo1"], value["beta"], value.get("state", "-"), value.get("elo_model", "BayesElo"), ) if name == "spsa" and value != "-": iter_local = value["iter"] + 1 # assume at least one completed, # and avoid division by zero A = value["A"] alpha = value["alpha"] gamma = value["gamma"] summary = "iter: {:d}, A: {:d}, alpha: {:0.3f}, gamma: {:0.3f}".format( iter_local, A, alpha, gamma, ) params = value["params"] value = [summary] for p in params: c_iter = p["c"] / (iter_local**gamma) r_iter = p["a"] / (A + iter_local)**alpha / c_iter**2 value.append([ p["name"], "{:.2f}".format(p["theta"]), int(p["start"]), int(p["min"]), int(p["max"]), "{:.3f}".format(c_iter), "{:.3f}".format(p["c_end"]), "{:.2e}".format(r_iter), "{:.2e}".format(p["r_end"]), ]) if "tests_repo" in run["args"]: if name == "new_tag": url = (run["args"]["tests_repo"] + "/commit/" + run["args"]["resolved_new"]) elif name == "base_tag": url = (run["args"]["tests_repo"] + "/commit/" + run["args"]["resolved_base"]) elif name == "tests_repo": url = value if name == "spsa": run_args.append(("spsa", value, "")) else: try: strval = str(value) except: strval = value.encode("ascii", "replace") if name not in ["new_tag", "base_tag"]: strval = html.escape(strval) run_args.append((name, strval, url)) active = 0 cores = 0 for task in run["tasks"]: if task["active"]: active += 1 cores += task["worker_info"]["concurrency"] last_updated = task.get("last_updated", datetime.datetime.min) task["last_updated"] = last_updated chi2 = get_chi2(run["tasks"]) update_residuals(run["tasks"], cached_chi2=chi2) return { "run": run, "run_args": run_args, "page_title": get_page_title(run), "approver": request.has_permission("approve_run"), "chi2": chi2, "totals": "({} active worker{} with {} core{})".format( active, ("s" if active != 1 else ""), cores, ("s" if cores != 1 else "")), "tasks_shown": request.cookies.get("tasks_state") == "Hide", }
def purge_run(self, run, p=0.001, res=7.0, iters=1): # Only purge finished runs assert run["finished"] now = datetime.utcnow() if "start_time" not in run or (now - run["start_time"]).days > 30: return "Run too old to be purged" # Do not revive failed runs if run.get("failed", False): return "You cannot purge a failed run" message = "No bad workers" # Transfer bad tasks to run["bad_tasks"] if "bad_tasks" not in run: run["bad_tasks"] = [] tasks = copy.copy(run["tasks"]) for task in tasks: # Special cases: crashes or time losses. if crash_or_time(task): message = "" # The next two lines are a bit hacky but # the correct residual and color may not have # been set yet. task["residual"] = 10.0 task["residual_color"] = "#FF6A6A" task["bad"] = True run["bad_tasks"].append(task) run["tasks"].remove(task) chi2 = get_chi2(run["tasks"]) # Make sure the residuals are up to date. # Once a task is moved to run["bad_tasks"] its # residual will no longer change. update_residuals(run["tasks"], cached_chi2=chi2) bad_workers = get_bad_workers( run["tasks"], cached_chi2=chi2, p=p, res=res, iters=iters - 1 if message == "" else iters, ) tasks = copy.copy(run["tasks"]) for task in tasks: if task["worker_info"]["unique_key"] in bad_workers: message = "" task["bad"] = True run["bad_tasks"].append(task) run["tasks"].remove(task) if message == "": run["results_stale"] = True results = self.get_results(run) revived = True if "sprt" in run["args"] and "state" in run["args"]["sprt"]: fishtest.stats.stat_util.update_SPRT(results, run["args"]["sprt"]) if run["args"]["sprt"]["state"] != "": revived = False run["results_info"] = format_results(results, run) if revived: run["finished"] = False run["is_green"] = False run["is_yellow"] = False else: # Copied code. Must be refactored. if run["results_info"]["style"] == "#44EB44": run["is_green"] = True elif run["results_info"]["style"] == "yellow": run["is_yellow"] = True self.buffer(run, True) return message