Esempio n. 1
0
    def setUp(self):
        global rundb
        rundb = RunDb()
        run_id = rundb.new_run('master',
                               'master',
                               100000,
                               '100+0.01',
                               'book',
                               10,
                               1,
                               '',
                               '',
                               username='******',
                               tests_repo='travis',
                               start_time=datetime.datetime.utcnow())
        json_params = {
            'username': '******',
            'password': '******',
            'run_id': run_id,
            'message': 'travis'
        }
        self.request = testing.DummyRequest(json_body=json_params,
                                            method='POST',
                                            rundb=rundb,
                                            userdb=rundb.userdb,
                                            actiondb=rundb.actiondb)

        config = testing.setUp(request=self.request)
Esempio n. 2
0
def update_users():
    rundb = RunDb()

    info = {}
    for u in rundb.userdb.get_users():
        username = u['username']
        info[username] = {
            'username': username,
            'cpu_hours': 0,
            'games': 0,
            'tests': 0,
            'tests_repo': u.get('tests_repo', ''),
            'last_updated': datetime.datetime.min,
            'games_per_hour': 0.0,
        }

    for run in rundb.get_runs():
        if 'deleted' in run:
            continue
        if 'username' in run['args']:
            username = run['args']['username']
            info[username]['tests'] += 1

        tc = parse_tc(run['args']['tc'])
        for task in run['tasks']:
            if 'worker_info' not in task:
                continue
            username = task['worker_info'].get('username', None)
            if username == None:
                continue

            if 'stats' in task:
                stats = task['stats']
                num_games = stats['wins'] + stats['losses'] + stats['draws']
            else:
                num_games = task['num_games']

            info[username]['last_updated'] = max(
                task['last_updated'], info[username]['last_updated'])
            info[username]['cpu_hours'] += float(num_games * tc / (60 * 60))
            info[username]['games'] += num_games

    machines = rundb.get_machines()
    for machine in machines:
        games_per_hour = (machine['nps'] / 1200000.0) * (3600.0 / parse_tc(
            machine['run']['args']['tc'])) * int(machine['concurrency'])
        info[machine['username']]['games_per_hour'] += games_per_hour

    users = []
    for u in info.keys():
        user = info[u]
        user['last_updated'] = delta_date(user['last_updated'])
        users.append(user)

    users = [u for u in users if u['games'] > 0 or u['tests'] > 0]

    rundb.userdb.user_cache.remove()
    rundb.userdb.user_cache.insert(users)
Esempio n. 3
0
def update_users():
  rundb = RunDb()

  info = {}
  for u in rundb.userdb.get_users():
    username = u['username']
    info[username] = {'username': username,
                      'cpu_hours': 0,
                      'games': 0,
                      'tests': 0,
                      'tests_repo': u.get('tests_repo', ''),
                      'last_updated': datetime.datetime.min,
                      'games_per_hour': 0.0,}

  for run in rundb.get_runs():
    if 'deleted' in run:
      continue
    if 'username' in run['args']:
      username = run['args']['username']
      info[username]['tests'] += 1

    tc = parse_tc(run['args']['tc'])
    for task in run['tasks']:
      if 'worker_info' not in task:
        continue
      username = task['worker_info'].get('username', None)
      if username == None:
        continue

      if 'stats' in task:
        stats = task['stats']
        num_games = stats['wins'] + stats['losses'] + stats['draws']
      else:
        num_games = task['num_games']

      info[username]['last_updated'] = max(task['last_updated'], info[username]['last_updated'])
      info[username]['cpu_hours'] += float(num_games * tc / (60 * 60))
      info[username]['games'] += num_games

  machines = rundb.get_machines()
  for machine in machines:
    games_per_hour = (machine['nps'] / 1200000.0) * (3600.0 / parse_tc(machine['run']['args']['tc'])) * int(machine['concurrency'])
    info[machine['username']]['games_per_hour'] += games_per_hour

  users = []
  for u in info.keys():
    user = info[u]
    user['last_updated'] = delta_date(user['last_updated'])
    users.append(user)

  users = [u for u in users if u['games'] > 0 or u['tests'] > 0]

  rundb.userdb.user_cache.remove()
  rundb.userdb.user_cache.insert(users)
Esempio n. 4
0
def scavenge_tasks(scavenge=True, minutes=5):
  """Check for tasks that have not been updated recently"""
  rundb = RunDb()
  for run in rundb.get_runs():
    changed = False
    for idx, task in enumerate(run['tasks']):
      if task['active'] and task['last_updated'] < datetime.utcnow() - timedelta(minutes=minutes):
        print 'Scavenging', task
        task['active'] = False
        rundb.clopdb.stop_games(str(run['_id']), idx)
        changed = True
    if changed and scavenge:
      rundb.runs.save(run)
Esempio n. 5
0
  def test_10_create_run(self):
    global rundb, run_id, run_id_stc
    rundb = RunDb()
    # STC
    run_id_stc = rundb.new_run('master', 'master', 100000, '10+0.01', 'book', 10, 1, '', '',
                               username='******', tests_repo='travis',
                               start_time=datetime.datetime.utcnow())
    run = rundb.get_run(run_id_stc)
    run['finished'] = True
    rundb.buffer(run, True)

    # LTC
    run_id = rundb.new_run('master', 'master', 100000, '150+0.01', 'book', 10, 1, '', '',
                           username='******', tests_repo='travis',
                           start_time=datetime.datetime.utcnow())
    print(' ')
    print(run_id)
    run = rundb.get_run(run_id)
    print(run['tasks'][0])
    self.assertFalse(run['tasks'][0][u'active'])
    run['tasks'][0][u'active'] = True
    run['tasks'][0][u'worker_info'] = {
      'username': '******', 'unique_key': 'travis', 'concurrency': 1}
    run['cores'] = 1

    print(util.find_run()['args'])
Esempio n. 6
0
  def setUp(self):
    global rundb
    rundb= RunDb()
    run_id = rundb.new_run('master', 'master', 100000, '100+0.01', 'book', 10, 1, '', '',
                           username='******', tests_repo='travis', start_time=datetime.datetime.utcnow())
    json_params= {'username': '******', 'password': '******', 'run_id': run_id, 'message': 'travis'}
    self.request = testing.DummyRequest(
            json_body=json_params,
            method='POST',
            rundb=rundb,
            userdb=rundb.userdb,
            actiondb=rundb.actiondb
            )

    config = testing.setUp(request=self.request)
Esempio n. 7
0
def update_users():
    rundb = RunDb()

    info = {}
    top_month = {}
    for u in rundb.userdb.get_users():
        username = u['username']
        info[username] = {
            'username': username,
            'cpu_hours': 0,
            'games': 0,
            'tests': 0,
            'tests_repo': u.get('tests_repo', ''),
            'last_updated': datetime.datetime.min,
            'games_per_hour': 0.0,
        }
        top_month[username] = info[username].copy()

    for run in rundb.get_unfinished_runs():
        process_run(run, info)
        process_run(run, top_month)

    # Step through these 100 at a time to avoid using too much RAM
    current = 0
    step_size = 100
    now = datetime.datetime.utcnow()
    while True:
        runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
        if len(runs) == 0:
            break
        for run in runs:
            process_run(run, info)
            if (now - run['start_time']).days < 31:
                process_run(run, top_month)
        current += step_size

    machines = rundb.get_machines()

    users = build_users(machines, info)
    rundb.userdb.user_cache.remove()
    rundb.userdb.user_cache.insert(users)

    rundb.userdb.top_month.remove()
    rundb.userdb.top_month.insert(build_users(machines, top_month))

    print('Successfully updated %d users' % (len(users)))
Esempio n. 8
0
  def setUp(self):
    global rundb
    rundb = RunDb()
    self.params = {'form.submitted': True, 'username': '******', 'password': '******'}
    self.request = testing.DummyRequest(
        params=self.params, userdb=rundb.userdb)

    config = testing.setUp(request=self.request)
Esempio n. 9
0
def update_users():
  rundb = RunDb()

  info = {}
  top_month = {}
  for u in rundb.userdb.get_users():
    username = u['username']
    info[username] = {'username': username,
                      'cpu_hours': 0,
                      'games': 0,
                      'tests': 0,
                      'tests_repo': u.get('tests_repo', ''),
                      'last_updated': datetime.datetime.min,
                      'games_per_hour': 0.0,}
    top_month[username] = info[username].copy()

  for run in rundb.get_unfinished_runs():
    process_run(run, info)
    process_run(run, top_month)

  # Step through these 100 at a time to avoid using too much RAM
  current = 0
  step_size = 100
  now = datetime.datetime.utcnow()
  while True:
    runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
    if len(runs) == 0:
      break
    for run in runs:
      process_run(run, info)
      if (now - run['start_time']).days < 31: 
        process_run(run, top_month)
    current += step_size

  machines = rundb.get_machines()

  users = build_users(machines, info)
  rundb.userdb.user_cache.remove()
  rundb.userdb.user_cache.insert(users)

  rundb.userdb.top_month.remove()
  rundb.userdb.top_month.insert(build_users(machines, top_month))

  print('Successfully updated %d users' % (len(users)))
Esempio n. 10
0
    def setUp(self):
        rundb = RunDb()
        self.request = testing.DummyRequest(params={
            'form.submitted': True,
            'username': '******',
            'password': '******',
            'email': '*****@*****.**'
        },
                                            userdb=rundb.userdb)

        config = testing.setUp(request=self.request)
Esempio n. 11
0
def scavenge_tasks(scavenge=True, minutes=60):
  """Check for tasks that have not been updated recently"""
  rundb = RunDb()
  for run in rundb.runs.find({'tasks': {'$elemMatch': {'active': True}}}):
    changed = False
    for idx, task in enumerate(run['tasks']):
      if task['active'] and task['last_updated'] < datetime.utcnow() - timedelta(minutes=minutes):
        print 'Scavenging', task
        task['active'] = False
        changed = True
    if changed and scavenge:
      rundb.runs.save(run)
Esempio n. 12
0
  def test_10_create_run(self): 
    global rundb, run_id, run_id_stc
    rundb= RunDb()
    # STC
    run_id_stc = rundb.new_run('master', 'master', 100000, '10+0.01', 'book', 10, 1, '', '',
                           username='******', tests_repo='travis', start_time=datetime.datetime.utcnow())
    run = rundb.get_run(run_id_stc)
    run['finished'] = True
    rundb.buffer(run, True)

    # LTC
    run_id = rundb.new_run('master', 'master', 100000, '150+0.01', 'book', 10, 1, '', '',
                           username='******', tests_repo='travis', start_time=datetime.datetime.utcnow())
    print(' '); print(run_id)
    run = rundb.get_run(run_id)
    print(run['tasks'][0])
    self.assertFalse(run['tasks'][0][u'active'])
    run['tasks'][0][u'active'] = True
    run['tasks'][0][u'worker_info'] = {'username': '******', 'unique_key': 'travis'}
    
    print(util.find_run()['args'])
Esempio n. 13
0
def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
  """
    session_factory = UnencryptedCookieSessionFactoryConfig('fishtest')
    config = Configurator(settings=settings,
                          session_factory=session_factory,
                          root_factory='fishtest.models.RootFactory')
    config.include('pyramid_mako')
    config.set_default_csrf_options(require_csrf=False)

    rundb = RunDb()

    def add_rundb(event):
        event.request.rundb = rundb
        event.request.userdb = rundb.userdb
        event.request.actiondb = rundb.actiondb

    def add_renderer_globals(event):
        event['h'] = helpers

    config.add_subscriber(add_rundb, NewRequest)
    config.add_subscriber(add_renderer_globals, BeforeRender)

    # Authentication
    def group_finder(username, request):
        return request.userdb.get_user_groups(username)

    with open(os.path.expanduser('~/fishtest.secret'), 'r') as f:
        secret = f.read()
    config.set_authentication_policy(
        AuthTktAuthenticationPolicy(secret,
                                    callback=group_finder,
                                    hashalg='sha512',
                                    http_only=True))
    config.set_authorization_policy(ACLAuthorizationPolicy())

    config.add_static_view('html', 'static/html', cache_max_age=3600)
    config.add_static_view('css', 'static/css', cache_max_age=3600)
    config.add_static_view('js', 'static/js', cache_max_age=3600)
    config.add_static_view('img', 'static/img', cache_max_age=3600)

    config.add_route('home', '/')
    config.add_route('login', '/login')
    config.add_route('nn_upload', '/upload')
    config.add_route('logout', '/logout')
    config.add_route('signup', '/signup')
    config.add_route('user', '/user/{username}')
    config.add_route('profile', '/user')
    config.add_route('pending', '/pending')
    config.add_route('users', '/users')
    config.add_route('users_monthly', '/users/monthly')
    config.add_route('actions', '/actions')
    config.add_route('nns', '/nns')

    config.add_route('tests', '/tests')
    config.add_route('tests_machines', '/tests/machines')
    config.add_route('tests_finished', '/tests/finished')
    config.add_route('tests_run', '/tests/run')
    config.add_route('tests_view', '/tests/view/{id}')
    config.add_route('tests_view_spsa_history',
                     '/tests/view/{id}/spsa_history')
    config.add_route('tests_user', '/tests/user/{username}')
    config.add_route('tests_stats', '/tests/stats/{id}')

    # Tests - actions
    config.add_route('tests_modify', '/tests/modify')
    config.add_route('tests_delete', '/tests/delete')
    config.add_route('tests_stop', '/tests/stop')
    config.add_route('tests_approve', '/tests/approve')
    config.add_route('tests_purge', '/tests/purge')

    # API
    config.add_route('api_request_task', '/api/request_task')
    config.add_route('api_update_task', '/api/update_task')
    config.add_route('api_failed_task', '/api/failed_task')
    config.add_route('api_stop_run', '/api/stop_run')
    config.add_route('api_request_version', '/api/request_version')
    config.add_route('api_request_spsa', '/api/request_spsa')
    config.add_route('api_active_runs', '/api/active_runs')
    config.add_route('api_get_run', '/api/get_run/{id}')
    config.add_route('api_upload_pgn', '/api/upload_pgn')
    config.add_route('api_download_pgn', '/api/pgn/{id}')
    config.add_route('api_download_pgn_100', '/api/pgn_100/{skip}')
    config.add_route('api_download_nn', '/api/nn/{id}')
    config.add_route('api_get_elo', '/api/get_elo/{id}')

    config.scan()
    return config.make_wsgi_app()
Esempio n. 14
0
#

from __future__ import print_function

import os
import sys
import pprint
import time
from pymongo import MongoClient, ASCENDING, DESCENDING

sys.path.append(os.path.expanduser('~/fishtest/fishtest'))
from fishtest.rundb import RunDb


db_name='fishtest_new'
rundb = RunDb()

# MongoDB server is assumed to be on the same machine, if not user should use
# ssh with port forwarding to access the remote host.
conn = MongoClient(os.getenv('FISHTEST_HOST') or 'localhost')
db = conn[db_name]
runs = db['runs']
pgns = db['pgns']



def printout(s):
  print(s)
  sys.stdout.flush()

Esempio n. 15
0
def main(global_config, **settings):
    """This function returns a Pyramid WSGI application."""
    session_factory = SignedCookieSessionFactory("fishtest")
    config = Configurator(
        settings=settings,
        session_factory=session_factory,
        root_factory="fishtest.models.RootFactory",
    )
    config.include("pyramid_mako")
    config.set_default_csrf_options(require_csrf=False)

    def static_file_full_path(static_file_path):
        return Path(__file__).parent / "./static/{}".format(static_file_path)

    def static_file_hash(static_file_path):
        with open(static_file_full_path(static_file_path), "r") as f:
            return hashlib.md5(f.read().encode("utf-8")).hexdigest()

    # the same hash calculated by browser for sub-resource integrity checks:
    # https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity
    dark_theme_sha256_integrity = (subprocess.run(
        "openssl dgst -sha256 -binary {} | openssl base64 -A".format(
            str(static_file_full_path(("css/theme.dark.css")))),
        shell=True,
        check=True,
        stdout=subprocess.PIPE,
    ).stdout.strip().decode("utf-8"))

    cache_busters = {
        "css/application.css": static_file_hash("css/application.css"),
        "css/theme.dark.css": dark_theme_sha256_integrity,
        "js/application.js": static_file_hash("js/application.js"),
    }

    rundb = RunDb()

    def add_rundb(event):
        event.request.rundb = rundb
        event.request.userdb = rundb.userdb
        event.request.actiondb = rundb.actiondb

    def add_renderer_globals(event):
        event["h"] = helpers
        event["cache_busters"] = cache_busters

    config.add_subscriber(add_rundb, NewRequest)
    config.add_subscriber(add_renderer_globals, BeforeRender)

    # Authentication
    def group_finder(username, request):
        return request.userdb.get_user_groups(username)

    with open(os.path.expanduser("~/fishtest.secret"), "r") as f:
        secret = f.read()
    config.set_authentication_policy(
        AuthTktAuthenticationPolicy(secret,
                                    callback=group_finder,
                                    hashalg="sha512",
                                    http_only=True))
    config.set_authorization_policy(ACLAuthorizationPolicy())

    config.add_static_view("html", "static/html", cache_max_age=3600)
    config.add_static_view("css", "static/css", cache_max_age=3600)
    config.add_static_view("js", "static/js", cache_max_age=3600)
    config.add_static_view("img", "static/img", cache_max_age=3600)

    config.add_route("home", "/")
    config.add_route("login", "/login")
    config.add_route("nn_upload", "/upload")
    config.add_route("logout", "/logout")
    config.add_route("signup", "/signup")
    config.add_route("user", "/user/{username}")
    config.add_route("profile", "/user")
    config.add_route("pending", "/pending")
    config.add_route("users", "/users")
    config.add_route("users_monthly", "/users/monthly")
    config.add_route("actions", "/actions")
    config.add_route("nns", "/nns")

    config.add_route("tests", "/tests")
    config.add_route("tests_machines", "/tests/machines")
    config.add_route("tests_finished", "/tests/finished")
    config.add_route("tests_run", "/tests/run")
    config.add_route("tests_view", "/tests/view/{id}")
    config.add_route("tests_view_spsa_history",
                     "/tests/view/{id}/spsa_history")
    config.add_route("tests_user", "/tests/user/{username}")
    config.add_route("tests_stats", "/tests/stats/{id}")

    # Tests - actions
    config.add_route("tests_modify", "/tests/modify")
    config.add_route("tests_delete", "/tests/delete")
    config.add_route("tests_stop", "/tests/stop")
    config.add_route("tests_approve", "/tests/approve")
    config.add_route("tests_purge", "/tests/purge")

    # API
    config.add_route("api_request_task", "/api/request_task")
    config.add_route("api_update_task", "/api/update_task")
    config.add_route("api_failed_task", "/api/failed_task")
    config.add_route("api_stop_run", "/api/stop_run")
    config.add_route("api_request_version", "/api/request_version")
    config.add_route("api_beat", "/api/beat")
    config.add_route("api_request_spsa", "/api/request_spsa")
    config.add_route("api_active_runs", "/api/active_runs")
    config.add_route("api_get_run", "/api/get_run/{id}")
    config.add_route("api_upload_pgn", "/api/upload_pgn")
    config.add_route("api_download_pgn", "/api/pgn/{id}")
    config.add_route("api_download_pgn_100", "/api/pgn_100/{skip}")
    config.add_route("api_download_nn", "/api/nn/{id}")
    config.add_route("api_get_elo", "/api/get_elo/{id}")

    config.scan()
    return config.make_wsgi_app()
Esempio n. 16
0
def main(global_config, **settings):
    """This function returns a Pyramid WSGI application."""
    session_factory = UnencryptedCookieSessionFactoryConfig("fishtest")
    config = Configurator(
        settings=settings,
        session_factory=session_factory,
        root_factory="fishtest.models.RootFactory",
    )
    config.include("pyramid_mako")
    config.set_default_csrf_options(require_csrf=False)

    rundb = RunDb()

    def add_rundb(event):
        event.request.rundb = rundb
        event.request.userdb = rundb.userdb
        event.request.actiondb = rundb.actiondb

    def add_renderer_globals(event):
        event["h"] = helpers

    config.add_subscriber(add_rundb, NewRequest)
    config.add_subscriber(add_renderer_globals, BeforeRender)

    # Authentication
    def group_finder(username, request):
        return request.userdb.get_user_groups(username)

    with open(os.path.expanduser("~/fishtest.secret"), "r") as f:
        secret = f.read()
    config.set_authentication_policy(
        AuthTktAuthenticationPolicy(secret,
                                    callback=group_finder,
                                    hashalg="sha512",
                                    http_only=True))
    config.set_authorization_policy(ACLAuthorizationPolicy())

    config.add_static_view("html", "static/html", cache_max_age=3600)
    config.add_static_view("css", "static/css", cache_max_age=3600)
    config.add_static_view("js", "static/js", cache_max_age=3600)
    config.add_static_view("img", "static/img", cache_max_age=3600)

    config.add_route("home", "/")
    config.add_route("login", "/login")
    config.add_route("nn_upload", "/upload")
    config.add_route("logout", "/logout")
    config.add_route("signup", "/signup")
    config.add_route("user", "/user/{username}")
    config.add_route("profile", "/user")
    config.add_route("pending", "/pending")
    config.add_route("users", "/users")
    config.add_route("users_monthly", "/users/monthly")
    config.add_route("actions", "/actions")
    config.add_route("nns", "/nns")

    config.add_route("tests", "/tests")
    config.add_route("tests_machines", "/tests/machines")
    config.add_route("tests_finished", "/tests/finished")
    config.add_route("tests_run", "/tests/run")
    config.add_route("tests_view", "/tests/view/{id}")
    config.add_route("tests_view_spsa_history",
                     "/tests/view/{id}/spsa_history")
    config.add_route("tests_user", "/tests/user/{username}")
    config.add_route("tests_stats", "/tests/stats/{id}")

    # Tests - actions
    config.add_route("tests_modify", "/tests/modify")
    config.add_route("tests_delete", "/tests/delete")
    config.add_route("tests_stop", "/tests/stop")
    config.add_route("tests_approve", "/tests/approve")
    config.add_route("tests_purge", "/tests/purge")

    # API
    config.add_route("api_request_task", "/api/request_task")
    config.add_route("api_update_task", "/api/update_task")
    config.add_route("api_failed_task", "/api/failed_task")
    config.add_route("api_stop_run", "/api/stop_run")
    config.add_route("api_request_version", "/api/request_version")
    config.add_route("api_request_spsa", "/api/request_spsa")
    config.add_route("api_active_runs", "/api/active_runs")
    config.add_route("api_get_run", "/api/get_run/{id}")
    config.add_route("api_upload_pgn", "/api/upload_pgn")
    config.add_route("api_download_pgn", "/api/pgn/{id}")
    config.add_route("api_download_pgn_100", "/api/pgn_100/{skip}")
    config.add_route("api_download_nn", "/api/nn/{id}")
    config.add_route("api_get_elo", "/api/get_elo/{id}")

    config.scan()
    return config.make_wsgi_app()
Esempio n. 17
0
def update_users():
    rundb = RunDb()

    deltas = {}
    info = {}
    top_month = {}

    clear_stats = True
    if len(sys.argv) > 1:
        print('scan all')
    else:
        deltas = rundb.deltas.find_one()
        if deltas:
            clear_stats = False
        else:
            deltas = {}

    for u in rundb.userdb.get_users():
        username = u['username']
        top_month[username] = {
            'username': username,
            'cpu_hours': 0,
            'games': 0,
            'tests': 0,
            'tests_repo': u.get('tests_repo', ''),
            'last_updated': datetime.min,
            'games_per_hour': 0.0,
        }
        if clear_stats:
            info[username] = top_month[username].copy()
        else:
            info[username] = rundb.userdb.user_cache.find_one(
                {'username': username})
            if not info[username]:
                info[username] = top_month[username].copy()
            else:
                info[username]['games_per_hour'] = 0.0

    for run in rundb.get_unfinished_runs():
        process_run(run, top_month)

    # Step through these in small batches (step size 100) to save RAM
    current = 0
    step_size = 100

    now = datetime.utcnow()
    more_days = True
    while more_days:
        runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
        if len(runs) == 0:
            break
        for run in runs:
            process_run(run, info, deltas)
            if (now - run['start_time']).days < 30:
                process_run(run, top_month)
            elif not clear_stats:
                more_days = False
        current += step_size

    if new_deltas:
        rundb.deltas.remove()
        rundb.deltas.save(new_deltas)

    machines = rundb.get_machines()

    users = build_users(machines, info)
    rundb.userdb.user_cache.remove()
    rundb.userdb.user_cache.insert(users)

    rundb.userdb.top_month.remove()
    rundb.userdb.top_month.insert(build_users(machines, top_month))

    # Delete users that have never been active and old admins group
    idle = {}
    for u in rundb.userdb.get_users():
        update = False
        while 'group:admins' in u['groups']:
            u['groups'].remove('group:admins')
            update = True
        if update:
            rundb.userdb.users.save(u)
        if not 'registration_time' in u \
           or u['registration_time'] < datetime.utcnow() - timedelta(days=28):
            idle[u['username']] = u
    for u in rundb.userdb.user_cache.find():
        if u['username'] in idle:
            del idle[u['username']]
    for u in idle.values():
        # A safe guard against deleting long time users
        if not 'registration_time' in u \
          or u['registration_time'] < datetime.utcnow() - timedelta(days=38):
            print('Warning: Found old user to delete: ' + str(u['_id']))
        else:
            print('Delete: ' + str(u['_id']))
            rundb.userdb.users.remove({'_id': u['_id']})

    print('Successfully updated %d users' % (len(users)))

    # record this update run
    rundb.actiondb.update_stats()
Esempio n. 18
0
def main(global_config, **settings):
    """This function returns a Pyramid WSGI application."""
    session_factory = SignedCookieSessionFactory("fishtest")
    config = Configurator(
        settings=settings,
        session_factory=session_factory,
        root_factory="fishtest.models.RootFactory",
    )
    config.include("pyramid_mako")
    config.set_default_csrf_options(require_csrf=False)

    def static_full_path(static_path):
        return Path(__file__).parent / f"static/{static_path}"

    def file_hash(file):
        return base64.b64encode(hashlib.sha384(
            file.read_bytes()).digest()).decode("utf8")

    # hash calculated by browser for sub-resource integrity checks:
    # https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity
    cache_busters = {
        f"{i}/{j.name}": file_hash(j)
        for i in ("css", "js") for j in static_full_path(i).glob(f"*.{i}")
    }

    rundb = RunDb()

    def add_rundb(event):
        event.request.rundb = rundb
        event.request.userdb = rundb.userdb
        event.request.actiondb = rundb.actiondb

    def add_renderer_globals(event):
        event["h"] = helpers
        event["cache_busters"] = cache_busters

    config.add_subscriber(add_rundb, NewRequest)
    config.add_subscriber(add_renderer_globals, BeforeRender)

    # Authentication
    def group_finder(username, request):
        return request.userdb.get_user_groups(username)

    secret = Path("~/fishtest.secret").expanduser().read_text()
    config.set_authentication_policy(
        AuthTktAuthenticationPolicy(secret,
                                    callback=group_finder,
                                    hashalg="sha512",
                                    http_only=True))
    config.set_authorization_policy(ACLAuthorizationPolicy())

    config.add_static_view("css", "static/css", cache_max_age=3600)
    config.add_static_view("js", "static/js", cache_max_age=3600)
    config.add_static_view("img", "static/img", cache_max_age=3600)

    config.add_route("home", "/")
    config.add_route("login", "/login")
    config.add_route("nn_upload", "/upload")
    config.add_route("logout", "/logout")
    config.add_route("signup", "/signup")
    config.add_route("user", "/user/{username}")
    config.add_route("profile", "/user")
    config.add_route("pending", "/pending")
    config.add_route("users", "/users")
    config.add_route("users_monthly", "/users/monthly")
    config.add_route("actions", "/actions")
    config.add_route("nns", "/nns")
    config.add_route("sprt_calc", "/sprt_calc")

    config.add_route("tests", "/tests")
    config.add_route("tests_finished", "/tests/finished")
    config.add_route("tests_run", "/tests/run")
    config.add_route("tests_view", "/tests/view/{id}")
    config.add_route("tests_user", "/tests/user/{username}")
    config.add_route("tests_stats", "/tests/stats/{id}")
    config.add_route("tests_live_elo", "/tests/live_elo/{id}")

    # Tests - actions
    config.add_route("tests_modify", "/tests/modify")
    config.add_route("tests_delete", "/tests/delete")
    config.add_route("tests_stop", "/tests/stop")
    config.add_route("tests_approve", "/tests/approve")
    config.add_route("tests_purge", "/tests/purge")

    # API
    config.add_route("api_request_task", "/api/request_task")
    config.add_route("api_update_task", "/api/update_task")
    config.add_route("api_failed_task", "/api/failed_task")
    config.add_route("api_stop_run", "/api/stop_run")
    config.add_route("api_request_version", "/api/request_version")
    config.add_route("api_beat", "/api/beat")
    config.add_route("api_request_spsa", "/api/request_spsa")
    config.add_route("api_active_runs", "/api/active_runs")
    config.add_route("api_get_run", "/api/get_run/{id}")
    config.add_route("api_get_task", "/api/get_task/{id}/{task_id}")
    config.add_route("api_upload_pgn", "/api/upload_pgn")
    config.add_route("api_download_pgn", "/api/pgn/{id}")
    config.add_route("api_download_pgn_100", "/api/pgn_100/{skip}")
    config.add_route("api_download_nn", "/api/nn/{id}")
    config.add_route("api_get_elo", "/api/get_elo/{id}")

    config.scan()
    return config.make_wsgi_app()
Esempio n. 19
0
def update_users():
    rundb = RunDb()

    info = {}
    for u in rundb.userdb.get_users():
        username = u["username"]
        info[username] = {
            "username": username,
            "cpu_hours": 0,
            "games": 0,
            "tests": 0,
            "tests_repo": u.get("tests_repo", ""),
            "last_updated": datetime.datetime.min,
            "games_per_hour": 0.0,
        }

    for u in rundb.userdb.old_user_cache.find():
        info[u["username"]] = u

    for run in rundb.get_runs():
        if "deleted" in run:
            continue
        if "username" in run["args"]:
            username = run["args"]["username"]
            info[username]["tests"] += 1

        tc = parse_tc(run["args"]["tc"])
        for task in run["tasks"]:
            if "worker_info" not in task:
                continue
            username = task["worker_info"].get("username", None)
            if username == None:
                continue

            if "stats" in task:
                stats = task["stats"]
                num_games = stats["wins"] + stats["losses"] + stats["draws"]
            else:
                num_games = task["num_games"]

            try:
                info[username]["last_updated"] = max(task["last_updated"], info[username]["last_updated"])
            except:
                info[username]["last_updated"] = task["last_updated"]

            info[username]["cpu_hours"] += float(num_games * tc / (60 * 60))
            info[username]["games"] += num_games

    machines = rundb.get_machines()
    for machine in machines:
        games_per_hour = (
            (machine["nps"] / 1200000.0)
            * (3600.0 / parse_tc(machine["run"]["args"]["tc"]))
            * int(machine["concurrency"])
        )
        info[machine["username"]]["games_per_hour"] += games_per_hour

    users = []
    for u in info.keys():
        user = info[u]
        try:
            user["last_updated"] = delta_date(user["last_updated"])
        except:
            pass
        users.append(user)

    users = [u for u in users if u["games"] > 0 or u["tests"] > 0]

    rundb.userdb.user_cache.remove()
    rundb.userdb.user_cache.insert(users)
Esempio n. 20
0
def update_users():
    rundb = RunDb()

    deltas = {}
    info = {}
    top_month = {}

    clear_stats = True
    if len(sys.argv) > 1:
        print("scan all")
    else:
        deltas = rundb.deltas.find_one()
        if deltas:
            clear_stats = False
        else:
            deltas = {}

    for u in rundb.userdb.get_users():
        username = u["username"]
        top_month[username] = {
            "username": username,
            "cpu_hours": 0,
            "games": 0,
            "tests": 0,
            "tests_repo": u.get("tests_repo", ""),
            "last_updated": datetime.min,
            "games_per_hour": 0.0,
        }
        if clear_stats:
            info[username] = top_month[username].copy()
        else:
            info[username] = rundb.userdb.user_cache.find_one({"username": username})
            if not info[username]:
                info[username] = top_month[username].copy()
            else:
                info[username]["games_per_hour"] = 0.0

    for run in rundb.get_unfinished_runs():
        try:
            process_run(run, top_month)
        except:
            print("Exception on run: ", run)

    # Step through these in small batches (step size 100) to save RAM
    step_size = 100

    now = datetime.utcnow()
    more_days = True
    last_updated = None
    while more_days:
        q = {"finished": True}
        if last_updated:
            q["last_updated"] = {"$lt": last_updated}
        runs = list(
            rundb.runs.find(q, sort=[("last_updated", DESCENDING)], limit=step_size)
        )
        if len(runs) == 0:
            break
        for run in runs:
            try:
                process_run(run, info, deltas)
            except:
                print("Exception on run: ", run["_id"])
            if (now - run["start_time"]).days < 30:
                try:
                    process_run(run, top_month)
                except:
                    print("Exception on run: ", run["_id"])
            elif not clear_stats:
                more_days = False
        last_updated = runs[-1]["last_updated"]

    if new_deltas:
        new_deltas.update(deltas)
        rundb.deltas.remove()
        rundb.deltas.save(new_deltas)

    machines = rundb.get_machines()

    users = build_users(machines, info)
    rundb.userdb.user_cache.remove()
    rundb.userdb.user_cache.insert(users)
    rundb.userdb.user_cache.create_index("username", unique=True)

    rundb.userdb.top_month.remove()
    rundb.userdb.top_month.insert(build_users(machines, top_month))

    # Delete users that have never been active and old admins group
    idle = {}
    for u in rundb.userdb.get_users():
        update = False
        while "group:admins" in u["groups"]:
            u["groups"].remove("group:admins")
            update = True
        if update:
            rundb.userdb.users.save(u)
        if not "registration_time" in u or u[
            "registration_time"
        ] < datetime.utcnow() - timedelta(days=28):
            idle[u["username"]] = u
    for u in rundb.userdb.user_cache.find():
        if u["username"] in idle:
            del idle[u["username"]]
    for u in idle.values():
        # A safe guard against deleting long time users
        if not "registration_time" in u or u[
            "registration_time"
        ] < datetime.utcnow() - timedelta(days=38):
            print("Warning: Found old user to delete: " + str(u["_id"]))
        else:
            print("Delete: " + str(u["_id"]))
            rundb.userdb.users.remove({"_id": u["_id"]})

    print("Successfully updated %d users" % (len(users)))

    # record this update run
    rundb.actiondb.update_stats()
Esempio n. 21
0
def find_run(arg='username', value='travis'):
  rundb= RunDb()
  for r in rundb.get_unfinished_runs():
    if r['args'][arg] == value:
      return r
  return None
Esempio n. 22
0
#

from __future__ import print_function

import os
import pprint
import sys
import time

from pymongo import ASCENDING, DESCENDING, MongoClient

sys.path.append(os.path.expanduser("~/fishtest/fishtest"))
from fishtest.rundb import RunDb

db_name = "fishtest_new"
rundb = RunDb()

# MongoDB server is assumed to be on the same machine, if not user should use
# ssh with port forwarding to access the remote host.
conn = MongoClient(os.getenv("FISHTEST_HOST") or "localhost")
db = conn[db_name]
runs = db["runs"]
pgns = db["pgns"]


def qlen(c):
    if c:
        return len(list(c))
    else:
        return 0
Esempio n. 23
0
def find_run(arg='username', value='travis'):
  rundb = RunDb()
  for run in rundb.get_unfinished_runs():
    if run['args'][arg] == value:
      return run
  return None
Esempio n. 24
0
def get_rundb():
    return RunDb(db_name="fishtest_tests")
Esempio n. 25
0
#!/usr/bin/python

from __future__ import print_function

import os, sys
import re
from datetime import datetime, timedelta

from fishtest.rundb import RunDb

rundb = RunDb()


def purge_pgn(days):
    """Purge old PGNs except LTC (>= 20s) runs"""

    deleted_runs = 0
    deleted_tasks = 0
    saved_runs = 0
    saved_tasks = 0
    now = datetime.utcnow()

    run_count = 0
    for run in rundb.runs.find({
            'finished': True,
            'deleted': {
                '$exists': False
            }
    }):

        if (now - run['start_time']).days > 30:
Esempio n. 26
0
def find_run(arg="username", value="travis"):
    rundb = RunDb(db_name="fishtest_tests")
    for run in rundb.get_unfinished_runs():
        if run["args"][arg] == value:
            return run
    return None