def stop_run(self, run_id):
"""Stops a run and runs auto-purge if it was enabled
- Used by the website and API for manually stopping runs
- Called during /api/update_task:
- for stopping SPRT runs if the test is accepted or rejected
- for stopping a run after all games are finished
"""
self.clear_params(run_id) # spsa stuff
run = self.get_run(run_id)
for task in run["tasks"]:
task["active"] = False
run["results_stale"] = True
results = self.get_results(run, True)
run["results_info"] = format_results(results, run)
# De-couple the styling of the run from its finished status
if run["results_info"]["style"] == "#44EB44":
run["is_green"] = True
elif run["results_info"]["style"] == "yellow":
run["is_yellow"] = True
run["finished"] = True
self.buffer(run, True)
# Publish the results of the run to the Fishcooking forum
post_in_fishcooking_results(run)
self.task_time = 0 # triggers a reload of self.task_runs
# Auto-purge runs here. This may revive the run.
if run["args"].get("auto_purge", True) and "spsa" not in run["args"]:
message = self.purge_run(run)
if message == "":
print("Run {} was auto-purged".format(str(run_id)), flush=True)
else:
print(
"Run {} was not auto-purged. Message: {}.".format(
str(run_id), message),
flush=True,
)
def aggregate_unfinished_runs(self, username=None):
unfinished_runs = self.get_unfinished_runs(username)
runs = {"pending": [], "active": []}
for run in unfinished_runs:
state = (
"active" if any(task["active"] for task in run["tasks"]) else "pending"
)
if state == "pending":
run["cores"] = 0
runs[state].append(run)
runs["pending"].sort(
key=lambda run: (
run["args"]["priority"],
run["args"]["itp"] if "itp" in run["args"] else 100,
)
)
runs["active"].sort(
reverse=True,
key=lambda run: (
"sprt" in run["args"],
run["args"].get("sprt", {}).get("llr", 0),
"spsa" not in run["args"],
run["results"]["wins"]
+ run["results"]["draws"]
+ run["results"]["losses"],
),
)
# Calculate but don't save results_info on runs using info on current machines
cores = 0
nps = 0
for m in self.get_machines():
concurrency = int(m["concurrency"])
cores += concurrency
nps += concurrency * m["nps"]
pending_hours = 0
for run in runs["pending"] + runs["active"]:
if cores > 0:
eta = remaining_hours(run) / cores
pending_hours += eta
results = self.get_results(run, False)
run["results_info"] = format_results(results, run)
if "Pending..." in run["results_info"]["info"]:
if cores > 0:
run["results_info"]["info"][0] += " (%.1f hrs)" % (eta)
if "sprt" in run["args"]:
sprt = run["args"]["sprt"]
elo_model = sprt.get("elo_model", "BayesElo")
if elo_model == "BayesElo":
run["results_info"]["info"].append(
("[%.2f,%.2f]") % (sprt["elo0"], sprt["elo1"])
)
else:
run["results_info"]["info"].append(
("{%.2f,%.2f}") % (sprt["elo0"], sprt["elo1"])
)
return (runs, pending_hours, cores, nps)
def stop_run(self, run_id, run=None):
""" Stops a run and runs auto-purge if it was enabled
- Used by the website and API for manually stopping runs
- Called during /api/update_task:
- for stopping SPRT runs if the test is accepted or rejected
- for stopping a run after all games are finished
"""
self.clear_params(run_id)
save_it = False
if run is None:
run = self.get_run(run_id)
save_it = True
run.pop('cores', None)
run['tasks'] = [task for task in run['tasks'] if 'stats' in task]
for task in run['tasks']:
task['pending'] = False
task['active'] = False
if save_it:
self.buffer(run, True)
self.task_time = 0
# Auto-purge runs here
purged = False
if run['args'].get('auto_purge', True) and 'spsa' not in run['args']:
if self.purge_run(run):
purged = True
run = self.get_run(run['_id'])
results = self.get_results(run, True)
run['results_info'] = format_results(results, run)
self.buffer(run, True)
if not purged:
# The run is now finished and will no longer be updated after this
run['finished'] = True
results = self.get_results(run, True)
run['results_info'] = format_results(results, run)
# De-couple the styling of the run from its finished status
if run['results_info']['style'] == '#44EB44':
run['is_green'] = True
elif run['results_info']['style'] == 'yellow':
run['is_yellow'] = True
self.buffer(run, True)
# Publish the results of the run to the Fishcooking forum
post_in_fishcooking_results(run)
def stop_run(self, run_id, run=None):
"""Stops a run and runs auto-purge if it was enabled
- Used by the website and API for manually stopping runs
- Called during /api/update_task:
- for stopping SPRT runs if the test is accepted or rejected
- for stopping a run after all games are finished
"""
self.clear_params(run_id)
save_it = False
if run is None:
run = self.get_run(run_id)
save_it = True
run["tasks"] = [task for task in run["tasks"] if "stats" in task]
for task in run["tasks"]:
task["pending"] = False
task["active"] = False
if save_it:
self.buffer(run, True)
self.task_time = 0
# Auto-purge runs here
purged = False
if run["args"].get("auto_purge",
True) and "spsa" not in run["args"]:
if self.purge_run(run):
purged = True
run = self.get_run(run["_id"])
results = self.get_results(run, True)
run["results_info"] = format_results(results, run)
self.buffer(run, True)
if not purged:
# The run is now finished and will no longer be updated after this
run["finished"] = True
results = self.get_results(run, True)
run["results_info"] = format_results(results, run)
# De-couple the styling of the run from its finished status
if run["results_info"]["style"] == "#44EB44":
run["is_green"] = True
elif run["results_info"]["style"] == "yellow":
run["is_yellow"] = True
self.buffer(run, True)
# Publish the results of the run to the Fishcooking forum
post_in_fishcooking_results(run)
def aggregate_unfinished_runs(self, username=None):
unfinished_runs = self.get_unfinished_runs(username)
runs = {'pending': [], 'active': []}
for run in unfinished_runs:
state = 'active' if any(task['active'] for task in run['tasks']) else 'pending'
runs[state].append(run)
runs['pending'].sort(key=lambda run: (run['args']['priority'],
run['args']['itp']
if 'itp' in run['args'] else 100))
runs['active'].sort(reverse=True, key=lambda run: (
'sprt' in run['args'],
run['args'].get('sprt',{}).get('llr',0),
'spsa' not in run['args'],
run['results']['wins'] + run['results']['draws']
+ run['results']['losses']))
# Calculate but don't save results_info on runs using info on current machines
cores = 0
nps = 0
for m in self.get_machines():
concurrency = int(m['concurrency'])
cores += concurrency
nps += concurrency * m['nps']
pending_hours = 0
for run in runs['pending'] + runs['active']:
if cores > 0:
eta = remaining_hours(run) / cores
pending_hours += eta
results = self.get_results(run, False)
run['results_info'] = format_results(results, run)
if 'Pending...' in run['results_info']['info']:
if cores > 0:
run['results_info']['info'][0] += ' (%.1f hrs)' % (eta)
if 'sprt' in run['args']:
sprt = run['args']['sprt']
elo_model = sprt.get('elo_model', 'BayesElo')
if elo_model == 'BayesElo':
run['results_info']['info'].append(('[%.2f,%.2f]')
% (sprt['elo0'], sprt['elo1']))
else:
run['results_info']['info'].append(('{%.2f,%.2f}')
% (sprt['elo0'], sprt['elo1']))
return (runs, pending_hours, cores, nps)
def get_paginated_finished_runs(request):
username = request.matchdict.get('username', '')
success_only = request.params.get('success_only', False)
yellow_only = request.params.get('yellow_only', False)
ltc_only = request.params.get('ltc_only', False)
page_idx = max(0, int(request.params.get('page', 1)) - 1)
page_size = 50
finished_runs, num_finished_runs = request.rundb.get_finished_runs(
username=username,
success_only=success_only,
yellow_only=yellow_only,
ltc_only=ltc_only,
skip=page_idx * page_size,
limit=page_size)
pages = [{
'idx': 'Prev',
'url': '?page={}'.format(page_idx),
'state': 'disabled' if page_idx == 0 else ''
}]
for idx, _ in enumerate(range(0, num_finished_runs, page_size)):
if idx < 5 or abs(page_idx - idx) < 5 or idx > (num_finished_runs /
page_size) - 5:
pages.append({
'idx': idx + 1,
'url': '?page={}'.format(idx + 1),
'state': 'active' if page_idx == idx else ''
})
elif pages[-1]['idx'] != '...':
pages.append({'idx': '...', 'url': '', 'state': 'disabled'})
pages.append({
'idx': 'Next',
'url': '?page={}'.format(page_idx + 2),
'state': 'disabled' if page_idx + 1 == len(pages) - 1 else ''
})
for page in pages:
if success_only:
page['url'] += '&success_only=1'
if yellow_only:
page['url'] += '&yellow_only=1'
if ltc_only:
page['url'] += '<c_only=1'
failed_runs = []
for run in finished_runs:
# Ensure finished runs have results_info
results = request.rundb.get_results(run)
if 'results_info' not in run:
run['results_info'] = format_results(results, run)
# Look for failed runs
if results['wins'] + results['losses'] + results['draws'] == 0:
failed_runs.append(run)
return {
'finished_runs': finished_runs,
'finished_runs_pages': pages,
'num_finished_runs': num_finished_runs,
'failed_runs': failed_runs,
'page_idx': page_idx,
}
def tests_view(request):
run = request.rundb.get_run(request.matchdict['id'])
if run is None:
raise exception_response(404)
results = request.rundb.get_results(run)
run['results_info'] = format_results(results, run)
run_args = [('id', str(run['_id']), '')]
for name in [
'new_tag', 'new_signature', 'new_options', 'resolved_new',
'base_tag', 'base_signature', 'base_options', 'resolved_base',
'sprt', 'num_games', 'spsa', 'tc', 'threads', 'book', 'book_depth',
'auto_purge', 'priority', 'itp', 'username', 'tests_repo', 'info'
]:
if name not in run['args']:
continue
value = run['args'][name]
url = ''
if name == 'new_tag' and 'msg_new' in run['args']:
value += ' (' + run['args']['msg_new'][:50] + ')'
if name == 'base_tag' and 'msg_base' in run['args']:
value += ' (' + run['args']['msg_base'][:50] + ')'
if name == 'sprt' and value != '-':
value = 'elo0: %.2f alpha: %.2f elo1: %.2f beta: %.2f state: %s (%s)' % \
(value['elo0'], value['alpha'], value['elo1'], value['beta'],
value.get('state', '-'), value.get('elo_model', 'BayesElo'))
if name == 'spsa' and value != '-':
iter_local = value['iter'] + 1 # assume at least one completed,
# and avoid division by zero
A = value['A']
alpha = value['alpha']
gamma = value['gamma']
summary = 'Iter: %d, A: %d, alpha %0.3f, gamma %0.3f, clipping %s, rounding %s' \
% (iter_local, A, alpha, gamma,
value['clipping'] if 'clipping' in value else 'old',
value['rounding'] if 'rounding' in value else 'deterministic')
params = value['params']
value = [summary]
for p in params:
value.append([
p['name'], '{:.2f}'.format(p['theta']),
int(p['start']),
int(p['min']),
int(p['max']),
'{:.3f}'.format(p['c'] / (iter_local**gamma)),
'{:.3f}'.format(p['a'] / (A + iter_local)**alpha)
])
if 'tests_repo' in run['args']:
if name == 'new_tag':
url = run['args']['tests_repo'] + '/commit/' + run['args'][
'resolved_new']
elif name == 'base_tag':
url = run['args']['tests_repo'] + '/commit/' + run['args'][
'resolved_base']
elif name == 'tests_repo':
url = value
if name == 'spsa':
run_args.append(('spsa', value, ''))
else:
try:
strval = str(value)
except:
strval = value.encode('ascii', 'replace')
strval = html.escape(strval)
run_args.append((name, strval, url))
active = 0
cores = 0
for task in run['tasks']:
if task['active']:
active += 1
cores += task['worker_info']['concurrency']
last_updated = task.get('last_updated', datetime.datetime.min)
task['last_updated'] = last_updated
if run['args'].get('sprt'):
page_title = 'SPRT {} vs {}'.format(run['args']['new_tag'],
run['args']['base_tag'])
elif run['args'].get('spsa'):
page_title = 'SPSA {}'.format(run['args']['new_tag'])
else:
page_title = '{} games - {} vs {}'.format(run['args']['num_games'],
run['args']['new_tag'],
run['args']['base_tag'])
return {
'run':
run,
'run_args':
run_args,
'page_title':
page_title,
'approver':
has_permission('approve_run', request.context, request),
'chi2':
calculate_residuals(run),
'totals':
'(%s active worker%s with %s core%s)' %
(active, ('s' if active != 1 else ''), cores,
('s' if cores != 1 else ''))
}
def tests_view(request):
run = request.rundb.get_run(request.matchdict["id"])
if run is None:
raise exception_response(404)
results = request.rundb.get_results(run)
run["results_info"] = format_results(results, run)
run_args = [("id", str(run["_id"]), "")]
if run.get("rescheduled_from"):
run_args.append(("rescheduled_from", run["rescheduled_from"], ""))
for name in [
"new_tag",
"new_signature",
"new_options",
"resolved_new",
"new_net",
"base_tag",
"base_signature",
"base_options",
"resolved_base",
"base_net",
"sprt",
"num_games",
"spsa",
"tc",
"threads",
"book",
"book_depth",
"auto_purge",
"priority",
"itp",
"username",
"tests_repo",
"info",
]:
if name not in run["args"]:
continue
value = run["args"][name]
url = ""
if name == "new_tag" and "msg_new" in run["args"]:
value += " (" + run["args"]["msg_new"][:50] + ")"
if name == "base_tag" and "msg_base" in run["args"]:
value += " (" + run["args"]["msg_base"][:50] + ")"
if name == "sprt" and value != "-":
value = "elo0: %.2f alpha: %.2f elo1: %.2f beta: %.2f state: %s (%s)" % (
value["elo0"],
value["alpha"],
value["elo1"],
value["beta"],
value.get("state", "-"),
value.get("elo_model", "BayesElo"),
)
if name == "spsa" and value != "-":
iter_local = value["iter"] + 1 # assume at least one completed,
# and avoid division by zero
A = value["A"]
alpha = value["alpha"]
gamma = value["gamma"]
summary = (
"Iter: %d, A: %d, alpha %0.3f, gamma %0.3f, clipping %s, rounding %s"
% (
iter_local,
A,
alpha,
gamma,
value["clipping"] if "clipping" in value else "old",
value["rounding"]
if "rounding" in value else "deterministic",
))
params = value["params"]
value = [summary]
for p in params:
value.append([
p["name"],
"{:.2f}".format(p["theta"]),
int(p["start"]),
int(p["min"]),
int(p["max"]),
"{:.3f}".format(p["c"] / (iter_local**gamma)),
"{:.3f}".format(p["a"] / (A + iter_local)**alpha),
])
if "tests_repo" in run["args"]:
if name == "new_tag":
url = (run["args"]["tests_repo"] + "/commit/" +
run["args"]["resolved_new"])
elif name == "base_tag":
url = (run["args"]["tests_repo"] + "/commit/" +
run["args"]["resolved_base"])
elif name == "tests_repo":
url = value
if name == "spsa":
run_args.append(("spsa", value, ""))
else:
try:
strval = str(value)
except:
strval = value.encode("ascii", "replace")
if name not in ["new_tag", "base_tag"]:
strval = html.escape(strval)
run_args.append((name, strval, url))
active = 0
cores = 0
for task in run["tasks"]:
if task["active"]:
active += 1
cores += task["worker_info"]["concurrency"]
last_updated = task.get("last_updated", datetime.datetime.min)
task["last_updated"] = last_updated
if run["args"].get("sprt"):
page_title = "SPRT {} vs {}".format(run["args"]["new_tag"],
run["args"]["base_tag"])
elif run["args"].get("spsa"):
page_title = "SPSA {}".format(run["args"]["new_tag"])
else:
page_title = "{} games - {} vs {}".format(run["args"]["num_games"],
run["args"]["new_tag"],
run["args"]["base_tag"])
return {
"run":
run,
"run_args":
run_args,
"page_title":
page_title,
"approver":
has_permission("approve_run", request.context, request),
"chi2":
calculate_residuals(run),
"totals":
"(%s active worker%s with %s core%s)" %
(active, ("s" if active != 1 else ""), cores,
("s" if cores != 1 else "")),
}
def get_paginated_finished_runs(request):
username = request.matchdict.get("username", "")
success_only = request.params.get("success_only", False)
yellow_only = request.params.get("yellow_only", False)
ltc_only = request.params.get("ltc_only", False)
page_idx = max(0, int(request.params.get("page", 1)) - 1)
page_size = 25
finished_runs, num_finished_runs = request.rundb.get_finished_runs(
username=username,
success_only=success_only,
yellow_only=yellow_only,
ltc_only=ltc_only,
skip=page_idx * page_size,
limit=page_size,
)
pages = [{
"idx": "Prev",
"url": "?page={}".format(page_idx),
"state": "disabled" if page_idx == 0 else "",
}]
for idx, _ in enumerate(range(0, num_finished_runs, page_size)):
if (idx < 5 or abs(page_idx - idx) < 5 or idx >
(num_finished_runs / page_size) - 5):
pages.append({
"idx": idx + 1,
"url": "?page={}".format(idx + 1),
"state": "active" if page_idx == idx else "",
})
elif pages[-1]["idx"] != "...":
pages.append({"idx": "...", "url": "", "state": "disabled"})
pages.append({
"idx": "Next",
"url": "?page={}".format(page_idx + 2),
"state": "disabled" if page_idx + 1 == len(pages) - 1 else "",
})
for page in pages:
if success_only:
page["url"] += "&success_only=1"
if yellow_only:
page["url"] += "&yellow_only=1"
if ltc_only:
page["url"] += "<c_only=1"
failed_runs = []
for run in finished_runs:
# Ensure finished runs have results_info
results = request.rundb.get_results(run)
if "results_info" not in run:
run["results_info"] = format_results(results, run)
# Look for failed runs
if "failed" in run:
failed_runs.append(run)
return {
"finished_runs": finished_runs,
"finished_runs_pages": pages,
"num_finished_runs": num_finished_runs,
"failed_runs": failed_runs,
"page_idx": page_idx,
}
def tests_view(request):
run = request.rundb.get_run(request.matchdict["id"])
if run is None:
raise exception_response(404)
results = request.rundb.get_results(run)
run["results_info"] = format_results(results, run)
run_args = [("id", str(run["_id"]), "")]
if run.get("rescheduled_from"):
run_args.append(("rescheduled_from", run["rescheduled_from"], ""))
for name in [
"new_tag",
"new_signature",
"new_options",
"resolved_new",
"new_net",
"base_tag",
"base_signature",
"base_options",
"resolved_base",
"base_net",
"sprt",
"num_games",
"spsa",
"tc",
"new_tc",
"threads",
"book",
"book_depth",
"auto_purge",
"priority",
"itp",
"username",
"tests_repo",
"adjudication",
"info",
]:
if name not in run["args"]:
continue
value = run["args"][name]
url = ""
if name == "new_tag" and "msg_new" in run["args"]:
value += " (" + run["args"]["msg_new"][:50] + ")"
if name == "base_tag" and "msg_base" in run["args"]:
value += " (" + run["args"]["msg_base"][:50] + ")"
if name == "sprt" and value != "-":
value = "elo0: {:.2f} alpha: {:.2f} elo1: {:.2f} beta: {:.2f} state: {} ({})".format(
value["elo0"],
value["alpha"],
value["elo1"],
value["beta"],
value.get("state", "-"),
value.get("elo_model", "BayesElo"),
)
if name == "spsa" and value != "-":
iter_local = value["iter"] + 1 # assume at least one completed,
# and avoid division by zero
A = value["A"]
alpha = value["alpha"]
gamma = value["gamma"]
summary = "iter: {:d}, A: {:d}, alpha: {:0.3f}, gamma: {:0.3f}".format(
iter_local,
A,
alpha,
gamma,
)
params = value["params"]
value = [summary]
for p in params:
c_iter = p["c"] / (iter_local**gamma)
r_iter = p["a"] / (A + iter_local)**alpha / c_iter**2
value.append([
p["name"],
"{:.2f}".format(p["theta"]),
int(p["start"]),
int(p["min"]),
int(p["max"]),
"{:.3f}".format(c_iter),
"{:.3f}".format(p["c_end"]),
"{:.2e}".format(r_iter),
"{:.2e}".format(p["r_end"]),
])
if "tests_repo" in run["args"]:
if name == "new_tag":
url = (run["args"]["tests_repo"] + "/commit/" +
run["args"]["resolved_new"])
elif name == "base_tag":
url = (run["args"]["tests_repo"] + "/commit/" +
run["args"]["resolved_base"])
elif name == "tests_repo":
url = value
if name == "spsa":
run_args.append(("spsa", value, ""))
else:
try:
strval = str(value)
except:
strval = value.encode("ascii", "replace")
if name not in ["new_tag", "base_tag"]:
strval = html.escape(strval)
run_args.append((name, strval, url))
active = 0
cores = 0
for task in run["tasks"]:
if task["active"]:
active += 1
cores += task["worker_info"]["concurrency"]
last_updated = task.get("last_updated", datetime.datetime.min)
task["last_updated"] = last_updated
chi2 = get_chi2(run["tasks"])
update_residuals(run["tasks"], cached_chi2=chi2)
return {
"run":
run,
"run_args":
run_args,
"page_title":
get_page_title(run),
"approver":
request.has_permission("approve_run"),
"chi2":
chi2,
"totals":
"({} active worker{} with {} core{})".format(
active, ("s" if active != 1 else ""), cores,
("s" if cores != 1 else "")),
"tasks_shown":
request.cookies.get("tasks_state") == "Hide",
}
def purge_run(self, run, p=0.001, res=7.0, iters=1):
# Only purge finished runs
assert run["finished"]
now = datetime.utcnow()
if "start_time" not in run or (now - run["start_time"]).days > 30:
return "Run too old to be purged"
# Do not revive failed runs
if run.get("failed", False):
return "You cannot purge a failed run"
message = "No bad workers"
# Transfer bad tasks to run["bad_tasks"]
if "bad_tasks" not in run:
run["bad_tasks"] = []
tasks = copy.copy(run["tasks"])
for task in tasks:
# Special cases: crashes or time losses.
if crash_or_time(task):
message = ""
# The next two lines are a bit hacky but
# the correct residual and color may not have
# been set yet.
task["residual"] = 10.0
task["residual_color"] = "#FF6A6A"
task["bad"] = True
run["bad_tasks"].append(task)
run["tasks"].remove(task)
chi2 = get_chi2(run["tasks"])
# Make sure the residuals are up to date.
# Once a task is moved to run["bad_tasks"] its
# residual will no longer change.
update_residuals(run["tasks"], cached_chi2=chi2)
bad_workers = get_bad_workers(
run["tasks"],
cached_chi2=chi2,
p=p,
res=res,
iters=iters - 1 if message == "" else iters,
)
tasks = copy.copy(run["tasks"])
for task in tasks:
if task["worker_info"]["unique_key"] in bad_workers:
message = ""
task["bad"] = True
run["bad_tasks"].append(task)
run["tasks"].remove(task)
if message == "":
run["results_stale"] = True
results = self.get_results(run)
revived = True
if "sprt" in run["args"] and "state" in run["args"]["sprt"]:
fishtest.stats.stat_util.update_SPRT(results,
run["args"]["sprt"])
if run["args"]["sprt"]["state"] != "":
revived = False
run["results_info"] = format_results(results, run)
if revived:
run["finished"] = False
run["is_green"] = False
run["is_yellow"] = False
else:
# Copied code. Must be refactored.
if run["results_info"]["style"] == "#44EB44":
run["is_green"] = True
elif run["results_info"]["style"] == "yellow":
run["is_yellow"] = True
self.buffer(run, True)
return message
