def setUp(self):
global rundb
rundb = RunDb()
run_id = rundb.new_run('master',
'master',
100000,
'100+0.01',
'book',
10,
1,
'',
'',
username='******',
tests_repo='travis',
start_time=datetime.datetime.utcnow())
json_params = {
'username': '******',
'password': '******',
'run_id': run_id,
'message': 'travis'
}
self.request = testing.DummyRequest(json_body=json_params,
method='POST',
rundb=rundb,
userdb=rundb.userdb,
actiondb=rundb.actiondb)
config = testing.setUp(request=self.request)
def update_users():
rundb = RunDb()
info = {}
for u in rundb.userdb.get_users():
username = u['username']
info[username] = {
'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.datetime.min,
'games_per_hour': 0.0,
}
for run in rundb.get_runs():
if 'deleted' in run:
continue
if 'username' in run['args']:
username = run['args']['username']
info[username]['tests'] += 1
tc = parse_tc(run['args']['tc'])
for task in run['tasks']:
if 'worker_info' not in task:
continue
username = task['worker_info'].get('username', None)
if username == None:
continue
if 'stats' in task:
stats = task['stats']
num_games = stats['wins'] + stats['losses'] + stats['draws']
else:
num_games = task['num_games']
info[username]['last_updated'] = max(
task['last_updated'], info[username]['last_updated'])
info[username]['cpu_hours'] += float(num_games * tc / (60 * 60))
info[username]['games'] += num_games
machines = rundb.get_machines()
for machine in machines:
games_per_hour = (machine['nps'] / 1200000.0) * (3600.0 / parse_tc(
machine['run']['args']['tc'])) * int(machine['concurrency'])
info[machine['username']]['games_per_hour'] += games_per_hour
users = []
for u in info.keys():
user = info[u]
user['last_updated'] = delta_date(user['last_updated'])
users.append(user)
users = [u for u in users if u['games'] > 0 or u['tests'] > 0]
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
def update_users():
rundb = RunDb()
info = {}
for u in rundb.userdb.get_users():
username = u['username']
info[username] = {'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.datetime.min,
'games_per_hour': 0.0,}
for run in rundb.get_runs():
if 'deleted' in run:
continue
if 'username' in run['args']:
username = run['args']['username']
info[username]['tests'] += 1
tc = parse_tc(run['args']['tc'])
for task in run['tasks']:
if 'worker_info' not in task:
continue
username = task['worker_info'].get('username', None)
if username == None:
continue
if 'stats' in task:
stats = task['stats']
num_games = stats['wins'] + stats['losses'] + stats['draws']
else:
num_games = task['num_games']
info[username]['last_updated'] = max(task['last_updated'], info[username]['last_updated'])
info[username]['cpu_hours'] += float(num_games * tc / (60 * 60))
info[username]['games'] += num_games
machines = rundb.get_machines()
for machine in machines:
games_per_hour = (machine['nps'] / 1200000.0) * (3600.0 / parse_tc(machine['run']['args']['tc'])) * int(machine['concurrency'])
info[machine['username']]['games_per_hour'] += games_per_hour
users = []
for u in info.keys():
user = info[u]
user['last_updated'] = delta_date(user['last_updated'])
users.append(user)
users = [u for u in users if u['games'] > 0 or u['tests'] > 0]
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
def scavenge_tasks(scavenge=True, minutes=5):
"""Check for tasks that have not been updated recently"""
rundb = RunDb()
for run in rundb.get_runs():
changed = False
for idx, task in enumerate(run['tasks']):
if task['active'] and task['last_updated'] < datetime.utcnow() - timedelta(minutes=minutes):
print 'Scavenging', task
task['active'] = False
rundb.clopdb.stop_games(str(run['_id']), idx)
changed = True
if changed and scavenge:
rundb.runs.save(run)
def test_10_create_run(self):
global rundb, run_id, run_id_stc
rundb = RunDb()
# STC
run_id_stc = rundb.new_run('master', 'master', 100000, '10+0.01', 'book', 10, 1, '', '',
username='******', tests_repo='travis',
start_time=datetime.datetime.utcnow())
run = rundb.get_run(run_id_stc)
run['finished'] = True
rundb.buffer(run, True)
# LTC
run_id = rundb.new_run('master', 'master', 100000, '150+0.01', 'book', 10, 1, '', '',
username='******', tests_repo='travis',
start_time=datetime.datetime.utcnow())
print(' ')
print(run_id)
run = rundb.get_run(run_id)
print(run['tasks'][0])
self.assertFalse(run['tasks'][0][u'active'])
run['tasks'][0][u'active'] = True
run['tasks'][0][u'worker_info'] = {
'username': '******', 'unique_key': 'travis', 'concurrency': 1}
run['cores'] = 1
print(util.find_run()['args'])
def setUp(self):
global rundb
rundb= RunDb()
run_id = rundb.new_run('master', 'master', 100000, '100+0.01', 'book', 10, 1, '', '',
username='******', tests_repo='travis', start_time=datetime.datetime.utcnow())
json_params= {'username': '******', 'password': '******', 'run_id': run_id, 'message': 'travis'}
self.request = testing.DummyRequest(
json_body=json_params,
method='POST',
rundb=rundb,
userdb=rundb.userdb,
actiondb=rundb.actiondb
)
config = testing.setUp(request=self.request)
def update_users():
rundb = RunDb()
info = {}
top_month = {}
for u in rundb.userdb.get_users():
username = u['username']
info[username] = {
'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.datetime.min,
'games_per_hour': 0.0,
}
top_month[username] = info[username].copy()
for run in rundb.get_unfinished_runs():
process_run(run, info)
process_run(run, top_month)
# Step through these 100 at a time to avoid using too much RAM
current = 0
step_size = 100
now = datetime.datetime.utcnow()
while True:
runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
if len(runs) == 0:
break
for run in runs:
process_run(run, info)
if (now - run['start_time']).days < 31:
process_run(run, top_month)
current += step_size
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
print('Successfully updated %d users' % (len(users)))
def setUp(self):
global rundb
rundb = RunDb()
self.params = {'form.submitted': True, 'username': '******', 'password': '******'}
self.request = testing.DummyRequest(
params=self.params, userdb=rundb.userdb)
config = testing.setUp(request=self.request)
def update_users():
rundb = RunDb()
info = {}
top_month = {}
for u in rundb.userdb.get_users():
username = u['username']
info[username] = {'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.datetime.min,
'games_per_hour': 0.0,}
top_month[username] = info[username].copy()
for run in rundb.get_unfinished_runs():
process_run(run, info)
process_run(run, top_month)
# Step through these 100 at a time to avoid using too much RAM
current = 0
step_size = 100
now = datetime.datetime.utcnow()
while True:
runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
if len(runs) == 0:
break
for run in runs:
process_run(run, info)
if (now - run['start_time']).days < 31:
process_run(run, top_month)
current += step_size
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
print('Successfully updated %d users' % (len(users)))
def setUp(self):
rundb = RunDb()
self.request = testing.DummyRequest(params={
'form.submitted': True,
'username': '******',
'password': '******',
'email': '*****@*****.**'
},
userdb=rundb.userdb)
config = testing.setUp(request=self.request)
def scavenge_tasks(scavenge=True, minutes=60):
"""Check for tasks that have not been updated recently"""
rundb = RunDb()
for run in rundb.runs.find({'tasks': {'$elemMatch': {'active': True}}}):
changed = False
for idx, task in enumerate(run['tasks']):
if task['active'] and task['last_updated'] < datetime.utcnow() - timedelta(minutes=minutes):
print 'Scavenging', task
task['active'] = False
changed = True
if changed and scavenge:
rundb.runs.save(run)
def test_10_create_run(self):
global rundb, run_id, run_id_stc
rundb= RunDb()
# STC
run_id_stc = rundb.new_run('master', 'master', 100000, '10+0.01', 'book', 10, 1, '', '',
username='******', tests_repo='travis', start_time=datetime.datetime.utcnow())
run = rundb.get_run(run_id_stc)
run['finished'] = True
rundb.buffer(run, True)
# LTC
run_id = rundb.new_run('master', 'master', 100000, '150+0.01', 'book', 10, 1, '', '',
username='******', tests_repo='travis', start_time=datetime.datetime.utcnow())
print(' '); print(run_id)
run = rundb.get_run(run_id)
print(run['tasks'][0])
self.assertFalse(run['tasks'][0][u'active'])
run['tasks'][0][u'active'] = True
run['tasks'][0][u'worker_info'] = {'username': '******', 'unique_key': 'travis'}
print(util.find_run()['args'])
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
session_factory = UnencryptedCookieSessionFactoryConfig('fishtest')
config = Configurator(settings=settings,
session_factory=session_factory,
root_factory='fishtest.models.RootFactory')
config.include('pyramid_mako')
config.set_default_csrf_options(require_csrf=False)
rundb = RunDb()
def add_rundb(event):
event.request.rundb = rundb
event.request.userdb = rundb.userdb
event.request.actiondb = rundb.actiondb
def add_renderer_globals(event):
event['h'] = helpers
config.add_subscriber(add_rundb, NewRequest)
config.add_subscriber(add_renderer_globals, BeforeRender)
# Authentication
def group_finder(username, request):
return request.userdb.get_user_groups(username)
with open(os.path.expanduser('~/fishtest.secret'), 'r') as f:
secret = f.read()
config.set_authentication_policy(
AuthTktAuthenticationPolicy(secret,
callback=group_finder,
hashalg='sha512',
http_only=True))
config.set_authorization_policy(ACLAuthorizationPolicy())
config.add_static_view('html', 'static/html', cache_max_age=3600)
config.add_static_view('css', 'static/css', cache_max_age=3600)
config.add_static_view('js', 'static/js', cache_max_age=3600)
config.add_static_view('img', 'static/img', cache_max_age=3600)
config.add_route('home', '/')
config.add_route('login', '/login')
config.add_route('nn_upload', '/upload')
config.add_route('logout', '/logout')
config.add_route('signup', '/signup')
config.add_route('user', '/user/{username}')
config.add_route('profile', '/user')
config.add_route('pending', '/pending')
config.add_route('users', '/users')
config.add_route('users_monthly', '/users/monthly')
config.add_route('actions', '/actions')
config.add_route('nns', '/nns')
config.add_route('tests', '/tests')
config.add_route('tests_machines', '/tests/machines')
config.add_route('tests_finished', '/tests/finished')
config.add_route('tests_run', '/tests/run')
config.add_route('tests_view', '/tests/view/{id}')
config.add_route('tests_view_spsa_history',
'/tests/view/{id}/spsa_history')
config.add_route('tests_user', '/tests/user/{username}')
config.add_route('tests_stats', '/tests/stats/{id}')
# Tests - actions
config.add_route('tests_modify', '/tests/modify')
config.add_route('tests_delete', '/tests/delete')
config.add_route('tests_stop', '/tests/stop')
config.add_route('tests_approve', '/tests/approve')
config.add_route('tests_purge', '/tests/purge')
# API
config.add_route('api_request_task', '/api/request_task')
config.add_route('api_update_task', '/api/update_task')
config.add_route('api_failed_task', '/api/failed_task')
config.add_route('api_stop_run', '/api/stop_run')
config.add_route('api_request_version', '/api/request_version')
config.add_route('api_request_spsa', '/api/request_spsa')
config.add_route('api_active_runs', '/api/active_runs')
config.add_route('api_get_run', '/api/get_run/{id}')
config.add_route('api_upload_pgn', '/api/upload_pgn')
config.add_route('api_download_pgn', '/api/pgn/{id}')
config.add_route('api_download_pgn_100', '/api/pgn_100/{skip}')
config.add_route('api_download_nn', '/api/nn/{id}')
config.add_route('api_get_elo', '/api/get_elo/{id}')
config.scan()
return config.make_wsgi_app()
#
from __future__ import print_function
import os
import sys
import pprint
import time
from pymongo import MongoClient, ASCENDING, DESCENDING
sys.path.append(os.path.expanduser('~/fishtest/fishtest'))
from fishtest.rundb import RunDb
db_name='fishtest_new'
rundb = RunDb()
# MongoDB server is assumed to be on the same machine, if not user should use
# ssh with port forwarding to access the remote host.
conn = MongoClient(os.getenv('FISHTEST_HOST') or 'localhost')
db = conn[db_name]
runs = db['runs']
pgns = db['pgns']
def printout(s):
print(s)
sys.stdout.flush()
def main(global_config, **settings):
"""This function returns a Pyramid WSGI application."""
session_factory = SignedCookieSessionFactory("fishtest")
config = Configurator(
settings=settings,
session_factory=session_factory,
root_factory="fishtest.models.RootFactory",
)
config.include("pyramid_mako")
config.set_default_csrf_options(require_csrf=False)
def static_file_full_path(static_file_path):
return Path(__file__).parent / "./static/{}".format(static_file_path)
def static_file_hash(static_file_path):
with open(static_file_full_path(static_file_path), "r") as f:
return hashlib.md5(f.read().encode("utf-8")).hexdigest()
# the same hash calculated by browser for sub-resource integrity checks:
# https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity
dark_theme_sha256_integrity = (subprocess.run(
"openssl dgst -sha256 -binary {} | openssl base64 -A".format(
str(static_file_full_path(("css/theme.dark.css")))),
shell=True,
check=True,
stdout=subprocess.PIPE,
).stdout.strip().decode("utf-8"))
cache_busters = {
"css/application.css": static_file_hash("css/application.css"),
"css/theme.dark.css": dark_theme_sha256_integrity,
"js/application.js": static_file_hash("js/application.js"),
}
rundb = RunDb()
def add_rundb(event):
event.request.rundb = rundb
event.request.userdb = rundb.userdb
event.request.actiondb = rundb.actiondb
def add_renderer_globals(event):
event["h"] = helpers
event["cache_busters"] = cache_busters
config.add_subscriber(add_rundb, NewRequest)
config.add_subscriber(add_renderer_globals, BeforeRender)
# Authentication
def group_finder(username, request):
return request.userdb.get_user_groups(username)
with open(os.path.expanduser("~/fishtest.secret"), "r") as f:
secret = f.read()
config.set_authentication_policy(
AuthTktAuthenticationPolicy(secret,
callback=group_finder,
hashalg="sha512",
http_only=True))
config.set_authorization_policy(ACLAuthorizationPolicy())
config.add_static_view("html", "static/html", cache_max_age=3600)
config.add_static_view("css", "static/css", cache_max_age=3600)
config.add_static_view("js", "static/js", cache_max_age=3600)
config.add_static_view("img", "static/img", cache_max_age=3600)
config.add_route("home", "/")
config.add_route("login", "/login")
config.add_route("nn_upload", "/upload")
config.add_route("logout", "/logout")
config.add_route("signup", "/signup")
config.add_route("user", "/user/{username}")
config.add_route("profile", "/user")
config.add_route("pending", "/pending")
config.add_route("users", "/users")
config.add_route("users_monthly", "/users/monthly")
config.add_route("actions", "/actions")
config.add_route("nns", "/nns")
config.add_route("tests", "/tests")
config.add_route("tests_machines", "/tests/machines")
config.add_route("tests_finished", "/tests/finished")
config.add_route("tests_run", "/tests/run")
config.add_route("tests_view", "/tests/view/{id}")
config.add_route("tests_view_spsa_history",
"/tests/view/{id}/spsa_history")
config.add_route("tests_user", "/tests/user/{username}")
config.add_route("tests_stats", "/tests/stats/{id}")
# Tests - actions
config.add_route("tests_modify", "/tests/modify")
config.add_route("tests_delete", "/tests/delete")
config.add_route("tests_stop", "/tests/stop")
config.add_route("tests_approve", "/tests/approve")
config.add_route("tests_purge", "/tests/purge")
# API
config.add_route("api_request_task", "/api/request_task")
config.add_route("api_update_task", "/api/update_task")
config.add_route("api_failed_task", "/api/failed_task")
config.add_route("api_stop_run", "/api/stop_run")
config.add_route("api_request_version", "/api/request_version")
config.add_route("api_beat", "/api/beat")
config.add_route("api_request_spsa", "/api/request_spsa")
config.add_route("api_active_runs", "/api/active_runs")
config.add_route("api_get_run", "/api/get_run/{id}")
config.add_route("api_upload_pgn", "/api/upload_pgn")
config.add_route("api_download_pgn", "/api/pgn/{id}")
config.add_route("api_download_pgn_100", "/api/pgn_100/{skip}")
config.add_route("api_download_nn", "/api/nn/{id}")
config.add_route("api_get_elo", "/api/get_elo/{id}")
config.scan()
return config.make_wsgi_app()
def main(global_config, **settings):
"""This function returns a Pyramid WSGI application."""
session_factory = UnencryptedCookieSessionFactoryConfig("fishtest")
config = Configurator(
settings=settings,
session_factory=session_factory,
root_factory="fishtest.models.RootFactory",
)
config.include("pyramid_mako")
config.set_default_csrf_options(require_csrf=False)
rundb = RunDb()
def add_rundb(event):
event.request.rundb = rundb
event.request.userdb = rundb.userdb
event.request.actiondb = rundb.actiondb
def add_renderer_globals(event):
event["h"] = helpers
config.add_subscriber(add_rundb, NewRequest)
config.add_subscriber(add_renderer_globals, BeforeRender)
# Authentication
def group_finder(username, request):
return request.userdb.get_user_groups(username)
with open(os.path.expanduser("~/fishtest.secret"), "r") as f:
secret = f.read()
config.set_authentication_policy(
AuthTktAuthenticationPolicy(secret,
callback=group_finder,
hashalg="sha512",
http_only=True))
config.set_authorization_policy(ACLAuthorizationPolicy())
config.add_static_view("html", "static/html", cache_max_age=3600)
config.add_static_view("css", "static/css", cache_max_age=3600)
config.add_static_view("js", "static/js", cache_max_age=3600)
config.add_static_view("img", "static/img", cache_max_age=3600)
config.add_route("home", "/")
config.add_route("login", "/login")
config.add_route("nn_upload", "/upload")
config.add_route("logout", "/logout")
config.add_route("signup", "/signup")
config.add_route("user", "/user/{username}")
config.add_route("profile", "/user")
config.add_route("pending", "/pending")
config.add_route("users", "/users")
config.add_route("users_monthly", "/users/monthly")
config.add_route("actions", "/actions")
config.add_route("nns", "/nns")
config.add_route("tests", "/tests")
config.add_route("tests_machines", "/tests/machines")
config.add_route("tests_finished", "/tests/finished")
config.add_route("tests_run", "/tests/run")
config.add_route("tests_view", "/tests/view/{id}")
config.add_route("tests_view_spsa_history",
"/tests/view/{id}/spsa_history")
config.add_route("tests_user", "/tests/user/{username}")
config.add_route("tests_stats", "/tests/stats/{id}")
# Tests - actions
config.add_route("tests_modify", "/tests/modify")
config.add_route("tests_delete", "/tests/delete")
config.add_route("tests_stop", "/tests/stop")
config.add_route("tests_approve", "/tests/approve")
config.add_route("tests_purge", "/tests/purge")
# API
config.add_route("api_request_task", "/api/request_task")
config.add_route("api_update_task", "/api/update_task")
config.add_route("api_failed_task", "/api/failed_task")
config.add_route("api_stop_run", "/api/stop_run")
config.add_route("api_request_version", "/api/request_version")
config.add_route("api_request_spsa", "/api/request_spsa")
config.add_route("api_active_runs", "/api/active_runs")
config.add_route("api_get_run", "/api/get_run/{id}")
config.add_route("api_upload_pgn", "/api/upload_pgn")
config.add_route("api_download_pgn", "/api/pgn/{id}")
config.add_route("api_download_pgn_100", "/api/pgn_100/{skip}")
config.add_route("api_download_nn", "/api/nn/{id}")
config.add_route("api_get_elo", "/api/get_elo/{id}")
config.scan()
return config.make_wsgi_app()
def update_users():
rundb = RunDb()
deltas = {}
info = {}
top_month = {}
clear_stats = True
if len(sys.argv) > 1:
print('scan all')
else:
deltas = rundb.deltas.find_one()
if deltas:
clear_stats = False
else:
deltas = {}
for u in rundb.userdb.get_users():
username = u['username']
top_month[username] = {
'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.min,
'games_per_hour': 0.0,
}
if clear_stats:
info[username] = top_month[username].copy()
else:
info[username] = rundb.userdb.user_cache.find_one(
{'username': username})
if not info[username]:
info[username] = top_month[username].copy()
else:
info[username]['games_per_hour'] = 0.0
for run in rundb.get_unfinished_runs():
process_run(run, top_month)
# Step through these in small batches (step size 100) to save RAM
current = 0
step_size = 100
now = datetime.utcnow()
more_days = True
while more_days:
runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
if len(runs) == 0:
break
for run in runs:
process_run(run, info, deltas)
if (now - run['start_time']).days < 30:
process_run(run, top_month)
elif not clear_stats:
more_days = False
current += step_size
if new_deltas:
rundb.deltas.remove()
rundb.deltas.save(new_deltas)
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
# Delete users that have never been active and old admins group
idle = {}
for u in rundb.userdb.get_users():
update = False
while 'group:admins' in u['groups']:
u['groups'].remove('group:admins')
update = True
if update:
rundb.userdb.users.save(u)
if not 'registration_time' in u \
or u['registration_time'] < datetime.utcnow() - timedelta(days=28):
idle[u['username']] = u
for u in rundb.userdb.user_cache.find():
if u['username'] in idle:
del idle[u['username']]
for u in idle.values():
# A safe guard against deleting long time users
if not 'registration_time' in u \
or u['registration_time'] < datetime.utcnow() - timedelta(days=38):
print('Warning: Found old user to delete: ' + str(u['_id']))
else:
print('Delete: ' + str(u['_id']))
rundb.userdb.users.remove({'_id': u['_id']})
print('Successfully updated %d users' % (len(users)))
# record this update run
rundb.actiondb.update_stats()
def main(global_config, **settings):
"""This function returns a Pyramid WSGI application."""
session_factory = SignedCookieSessionFactory("fishtest")
config = Configurator(
settings=settings,
session_factory=session_factory,
root_factory="fishtest.models.RootFactory",
)
config.include("pyramid_mako")
config.set_default_csrf_options(require_csrf=False)
def static_full_path(static_path):
return Path(__file__).parent / f"static/{static_path}"
def file_hash(file):
return base64.b64encode(hashlib.sha384(
file.read_bytes()).digest()).decode("utf8")
# hash calculated by browser for sub-resource integrity checks:
# https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity
cache_busters = {
f"{i}/{j.name}": file_hash(j)
for i in ("css", "js") for j in static_full_path(i).glob(f"*.{i}")
}
rundb = RunDb()
def add_rundb(event):
event.request.rundb = rundb
event.request.userdb = rundb.userdb
event.request.actiondb = rundb.actiondb
def add_renderer_globals(event):
event["h"] = helpers
event["cache_busters"] = cache_busters
config.add_subscriber(add_rundb, NewRequest)
config.add_subscriber(add_renderer_globals, BeforeRender)
# Authentication
def group_finder(username, request):
return request.userdb.get_user_groups(username)
secret = Path("~/fishtest.secret").expanduser().read_text()
config.set_authentication_policy(
AuthTktAuthenticationPolicy(secret,
callback=group_finder,
hashalg="sha512",
http_only=True))
config.set_authorization_policy(ACLAuthorizationPolicy())
config.add_static_view("css", "static/css", cache_max_age=3600)
config.add_static_view("js", "static/js", cache_max_age=3600)
config.add_static_view("img", "static/img", cache_max_age=3600)
config.add_route("home", "/")
config.add_route("login", "/login")
config.add_route("nn_upload", "/upload")
config.add_route("logout", "/logout")
config.add_route("signup", "/signup")
config.add_route("user", "/user/{username}")
config.add_route("profile", "/user")
config.add_route("pending", "/pending")
config.add_route("users", "/users")
config.add_route("users_monthly", "/users/monthly")
config.add_route("actions", "/actions")
config.add_route("nns", "/nns")
config.add_route("sprt_calc", "/sprt_calc")
config.add_route("tests", "/tests")
config.add_route("tests_finished", "/tests/finished")
config.add_route("tests_run", "/tests/run")
config.add_route("tests_view", "/tests/view/{id}")
config.add_route("tests_user", "/tests/user/{username}")
config.add_route("tests_stats", "/tests/stats/{id}")
config.add_route("tests_live_elo", "/tests/live_elo/{id}")
# Tests - actions
config.add_route("tests_modify", "/tests/modify")
config.add_route("tests_delete", "/tests/delete")
config.add_route("tests_stop", "/tests/stop")
config.add_route("tests_approve", "/tests/approve")
config.add_route("tests_purge", "/tests/purge")
# API
config.add_route("api_request_task", "/api/request_task")
config.add_route("api_update_task", "/api/update_task")
config.add_route("api_failed_task", "/api/failed_task")
config.add_route("api_stop_run", "/api/stop_run")
config.add_route("api_request_version", "/api/request_version")
config.add_route("api_beat", "/api/beat")
config.add_route("api_request_spsa", "/api/request_spsa")
config.add_route("api_active_runs", "/api/active_runs")
config.add_route("api_get_run", "/api/get_run/{id}")
config.add_route("api_get_task", "/api/get_task/{id}/{task_id}")
config.add_route("api_upload_pgn", "/api/upload_pgn")
config.add_route("api_download_pgn", "/api/pgn/{id}")
config.add_route("api_download_pgn_100", "/api/pgn_100/{skip}")
config.add_route("api_download_nn", "/api/nn/{id}")
config.add_route("api_get_elo", "/api/get_elo/{id}")
config.scan()
return config.make_wsgi_app()
def update_users():
rundb = RunDb()
info = {}
for u in rundb.userdb.get_users():
username = u["username"]
info[username] = {
"username": username,
"cpu_hours": 0,
"games": 0,
"tests": 0,
"tests_repo": u.get("tests_repo", ""),
"last_updated": datetime.datetime.min,
"games_per_hour": 0.0,
}
for u in rundb.userdb.old_user_cache.find():
info[u["username"]] = u
for run in rundb.get_runs():
if "deleted" in run:
continue
if "username" in run["args"]:
username = run["args"]["username"]
info[username]["tests"] += 1
tc = parse_tc(run["args"]["tc"])
for task in run["tasks"]:
if "worker_info" not in task:
continue
username = task["worker_info"].get("username", None)
if username == None:
continue
if "stats" in task:
stats = task["stats"]
num_games = stats["wins"] + stats["losses"] + stats["draws"]
else:
num_games = task["num_games"]
try:
info[username]["last_updated"] = max(task["last_updated"], info[username]["last_updated"])
except:
info[username]["last_updated"] = task["last_updated"]
info[username]["cpu_hours"] += float(num_games * tc / (60 * 60))
info[username]["games"] += num_games
machines = rundb.get_machines()
for machine in machines:
games_per_hour = (
(machine["nps"] / 1200000.0)
* (3600.0 / parse_tc(machine["run"]["args"]["tc"]))
* int(machine["concurrency"])
)
info[machine["username"]]["games_per_hour"] += games_per_hour
users = []
for u in info.keys():
user = info[u]
try:
user["last_updated"] = delta_date(user["last_updated"])
except:
pass
users.append(user)
users = [u for u in users if u["games"] > 0 or u["tests"] > 0]
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
def update_users():
rundb = RunDb()
deltas = {}
info = {}
top_month = {}
clear_stats = True
if len(sys.argv) > 1:
print("scan all")
else:
deltas = rundb.deltas.find_one()
if deltas:
clear_stats = False
else:
deltas = {}
for u in rundb.userdb.get_users():
username = u["username"]
top_month[username] = {
"username": username,
"cpu_hours": 0,
"games": 0,
"tests": 0,
"tests_repo": u.get("tests_repo", ""),
"last_updated": datetime.min,
"games_per_hour": 0.0,
}
if clear_stats:
info[username] = top_month[username].copy()
else:
info[username] = rundb.userdb.user_cache.find_one({"username": username})
if not info[username]:
info[username] = top_month[username].copy()
else:
info[username]["games_per_hour"] = 0.0
for run in rundb.get_unfinished_runs():
try:
process_run(run, top_month)
except:
print("Exception on run: ", run)
# Step through these in small batches (step size 100) to save RAM
step_size = 100
now = datetime.utcnow()
more_days = True
last_updated = None
while more_days:
q = {"finished": True}
if last_updated:
q["last_updated"] = {"$lt": last_updated}
runs = list(
rundb.runs.find(q, sort=[("last_updated", DESCENDING)], limit=step_size)
)
if len(runs) == 0:
break
for run in runs:
try:
process_run(run, info, deltas)
except:
print("Exception on run: ", run["_id"])
if (now - run["start_time"]).days < 30:
try:
process_run(run, top_month)
except:
print("Exception on run: ", run["_id"])
elif not clear_stats:
more_days = False
last_updated = runs[-1]["last_updated"]
if new_deltas:
new_deltas.update(deltas)
rundb.deltas.remove()
rundb.deltas.save(new_deltas)
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.user_cache.create_index("username", unique=True)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
# Delete users that have never been active and old admins group
idle = {}
for u in rundb.userdb.get_users():
update = False
while "group:admins" in u["groups"]:
u["groups"].remove("group:admins")
update = True
if update:
rundb.userdb.users.save(u)
if not "registration_time" in u or u[
"registration_time"
] < datetime.utcnow() - timedelta(days=28):
idle[u["username"]] = u
for u in rundb.userdb.user_cache.find():
if u["username"] in idle:
del idle[u["username"]]
for u in idle.values():
# A safe guard against deleting long time users
if not "registration_time" in u or u[
"registration_time"
] < datetime.utcnow() - timedelta(days=38):
print("Warning: Found old user to delete: " + str(u["_id"]))
else:
print("Delete: " + str(u["_id"]))
rundb.userdb.users.remove({"_id": u["_id"]})
print("Successfully updated %d users" % (len(users)))
# record this update run
rundb.actiondb.update_stats()
def find_run(arg='username', value='travis'):
rundb= RunDb()
for r in rundb.get_unfinished_runs():
if r['args'][arg] == value:
return r
return None
#
from __future__ import print_function
import os
import pprint
import sys
import time
from pymongo import ASCENDING, DESCENDING, MongoClient
sys.path.append(os.path.expanduser("~/fishtest/fishtest"))
from fishtest.rundb import RunDb
db_name = "fishtest_new"
rundb = RunDb()
# MongoDB server is assumed to be on the same machine, if not user should use
# ssh with port forwarding to access the remote host.
conn = MongoClient(os.getenv("FISHTEST_HOST") or "localhost")
db = conn[db_name]
runs = db["runs"]
pgns = db["pgns"]
def qlen(c):
if c:
return len(list(c))
else:
return 0
def find_run(arg='username', value='travis'):
rundb = RunDb()
for run in rundb.get_unfinished_runs():
if run['args'][arg] == value:
return run
return None
def get_rundb():
return RunDb(db_name="fishtest_tests")
#!/usr/bin/python
from __future__ import print_function
import os, sys
import re
from datetime import datetime, timedelta
from fishtest.rundb import RunDb
rundb = RunDb()
def purge_pgn(days):
"""Purge old PGNs except LTC (>= 20s) runs"""
deleted_runs = 0
deleted_tasks = 0
saved_runs = 0
saved_tasks = 0
now = datetime.utcnow()
run_count = 0
for run in rundb.runs.find({
'finished': True,
'deleted': {
'$exists': False
}
}):
if (now - run['start_time']).days > 30:
def find_run(arg="username", value="travis"):
rundb = RunDb(db_name="fishtest_tests")
for run in rundb.get_unfinished_runs():
if run["args"][arg] == value:
return run
return None
