예제 #1
0
def update_users():
    rundb = RunDb()

    info = {}
    for u in rundb.userdb.get_users():
        username = u['username']
        info[username] = {
            'username': username,
            'cpu_hours': 0,
            'games': 0,
            'tests': 0,
            'tests_repo': u.get('tests_repo', ''),
            'last_updated': datetime.datetime.min,
            'games_per_hour': 0.0,
        }

    for run in rundb.get_runs():
        if 'deleted' in run:
            continue
        if 'username' in run['args']:
            username = run['args']['username']
            info[username]['tests'] += 1

        tc = parse_tc(run['args']['tc'])
        for task in run['tasks']:
            if 'worker_info' not in task:
                continue
            username = task['worker_info'].get('username', None)
            if username == None:
                continue

            if 'stats' in task:
                stats = task['stats']
                num_games = stats['wins'] + stats['losses'] + stats['draws']
            else:
                num_games = task['num_games']

            info[username]['last_updated'] = max(
                task['last_updated'], info[username]['last_updated'])
            info[username]['cpu_hours'] += float(num_games * tc / (60 * 60))
            info[username]['games'] += num_games

    machines = rundb.get_machines()
    for machine in machines:
        games_per_hour = (machine['nps'] / 1200000.0) * (3600.0 / parse_tc(
            machine['run']['args']['tc'])) * int(machine['concurrency'])
        info[machine['username']]['games_per_hour'] += games_per_hour

    users = []
    for u in info.keys():
        user = info[u]
        user['last_updated'] = delta_date(user['last_updated'])
        users.append(user)

    users = [u for u in users if u['games'] > 0 or u['tests'] > 0]

    rundb.userdb.user_cache.remove()
    rundb.userdb.user_cache.insert(users)
예제 #2
0
def update_users():
  rundb = RunDb()

  info = {}
  for u in rundb.userdb.get_users():
    username = u['username']
    info[username] = {'username': username,
                      'cpu_hours': 0,
                      'games': 0,
                      'tests': 0,
                      'tests_repo': u.get('tests_repo', ''),
                      'last_updated': datetime.datetime.min,
                      'games_per_hour': 0.0,}

  for run in rundb.get_runs():
    if 'deleted' in run:
      continue
    if 'username' in run['args']:
      username = run['args']['username']
      info[username]['tests'] += 1

    tc = parse_tc(run['args']['tc'])
    for task in run['tasks']:
      if 'worker_info' not in task:
        continue
      username = task['worker_info'].get('username', None)
      if username == None:
        continue

      if 'stats' in task:
        stats = task['stats']
        num_games = stats['wins'] + stats['losses'] + stats['draws']
      else:
        num_games = task['num_games']

      info[username]['last_updated'] = max(task['last_updated'], info[username]['last_updated'])
      info[username]['cpu_hours'] += float(num_games * tc / (60 * 60))
      info[username]['games'] += num_games

  machines = rundb.get_machines()
  for machine in machines:
    games_per_hour = (machine['nps'] / 1200000.0) * (3600.0 / parse_tc(machine['run']['args']['tc'])) * int(machine['concurrency'])
    info[machine['username']]['games_per_hour'] += games_per_hour

  users = []
  for u in info.keys():
    user = info[u]
    user['last_updated'] = delta_date(user['last_updated'])
    users.append(user)

  users = [u for u in users if u['games'] > 0 or u['tests'] > 0]

  rundb.userdb.user_cache.remove()
  rundb.userdb.user_cache.insert(users)
예제 #3
0
def update_users():
    rundb = RunDb()

    info = {}
    top_month = {}
    for u in rundb.userdb.get_users():
        username = u['username']
        info[username] = {
            'username': username,
            'cpu_hours': 0,
            'games': 0,
            'tests': 0,
            'tests_repo': u.get('tests_repo', ''),
            'last_updated': datetime.datetime.min,
            'games_per_hour': 0.0,
        }
        top_month[username] = info[username].copy()

    for run in rundb.get_unfinished_runs():
        process_run(run, info)
        process_run(run, top_month)

    # Step through these 100 at a time to avoid using too much RAM
    current = 0
    step_size = 100
    now = datetime.datetime.utcnow()
    while True:
        runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
        if len(runs) == 0:
            break
        for run in runs:
            process_run(run, info)
            if (now - run['start_time']).days < 31:
                process_run(run, top_month)
        current += step_size

    machines = rundb.get_machines()

    users = build_users(machines, info)
    rundb.userdb.user_cache.remove()
    rundb.userdb.user_cache.insert(users)

    rundb.userdb.top_month.remove()
    rundb.userdb.top_month.insert(build_users(machines, top_month))

    print('Successfully updated %d users' % (len(users)))
예제 #4
0
def update_users():
  rundb = RunDb()

  info = {}
  top_month = {}
  for u in rundb.userdb.get_users():
    username = u['username']
    info[username] = {'username': username,
                      'cpu_hours': 0,
                      'games': 0,
                      'tests': 0,
                      'tests_repo': u.get('tests_repo', ''),
                      'last_updated': datetime.datetime.min,
                      'games_per_hour': 0.0,}
    top_month[username] = info[username].copy()

  for run in rundb.get_unfinished_runs():
    process_run(run, info)
    process_run(run, top_month)

  # Step through these 100 at a time to avoid using too much RAM
  current = 0
  step_size = 100
  now = datetime.datetime.utcnow()
  while True:
    runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
    if len(runs) == 0:
      break
    for run in runs:
      process_run(run, info)
      if (now - run['start_time']).days < 31: 
        process_run(run, top_month)
    current += step_size

  machines = rundb.get_machines()

  users = build_users(machines, info)
  rundb.userdb.user_cache.remove()
  rundb.userdb.user_cache.insert(users)

  rundb.userdb.top_month.remove()
  rundb.userdb.top_month.insert(build_users(machines, top_month))

  print('Successfully updated %d users' % (len(users)))
예제 #5
0
def update_users():
    rundb = RunDb()

    info = {}
    for u in rundb.userdb.get_users():
        username = u["username"]
        info[username] = {
            "username": username,
            "cpu_hours": 0,
            "games": 0,
            "tests": 0,
            "tests_repo": u.get("tests_repo", ""),
            "last_updated": datetime.datetime.min,
            "games_per_hour": 0.0,
        }

    for u in rundb.userdb.old_user_cache.find():
        info[u["username"]] = u

    for run in rundb.get_runs():
        if "deleted" in run:
            continue
        if "username" in run["args"]:
            username = run["args"]["username"]
            info[username]["tests"] += 1

        tc = parse_tc(run["args"]["tc"])
        for task in run["tasks"]:
            if "worker_info" not in task:
                continue
            username = task["worker_info"].get("username", None)
            if username == None:
                continue

            if "stats" in task:
                stats = task["stats"]
                num_games = stats["wins"] + stats["losses"] + stats["draws"]
            else:
                num_games = task["num_games"]

            try:
                info[username]["last_updated"] = max(task["last_updated"], info[username]["last_updated"])
            except:
                info[username]["last_updated"] = task["last_updated"]

            info[username]["cpu_hours"] += float(num_games * tc / (60 * 60))
            info[username]["games"] += num_games

    machines = rundb.get_machines()
    for machine in machines:
        games_per_hour = (
            (machine["nps"] / 1200000.0)
            * (3600.0 / parse_tc(machine["run"]["args"]["tc"]))
            * int(machine["concurrency"])
        )
        info[machine["username"]]["games_per_hour"] += games_per_hour

    users = []
    for u in info.keys():
        user = info[u]
        try:
            user["last_updated"] = delta_date(user["last_updated"])
        except:
            pass
        users.append(user)

    users = [u for u in users if u["games"] > 0 or u["tests"] > 0]

    rundb.userdb.user_cache.remove()
    rundb.userdb.user_cache.insert(users)
예제 #6
0

def printout(s):
  print(s)
  sys.stdout.flush()



printout("\nFetching unfinished runs ...")
start = time.time()
unfinished_runs = rundb.get_unfinished_runs()
end = time.time()

printout(str(end-start) + "s\nFetching machines ...")
start = time.time()
machines = rundb.get_machines()
end = time.time()

printout(str(end-start) + "s\nFetching finished runs ...")
start = time.time()
finished, num_finished = rundb.get_finished_runs(skip=0, limit=50, username='',
                                                 success_only=False, ltc_only=False)
end = time.time()

printout(str(end-start) + "s\nRequesting pgn ...")
if (len(finished) == 0):
    finished.append({'_id':'abc'})
start = time.time()
pgn = rundb.get_pgn(str(finished[0]['_id']) + ".pgn")
end = time.time()
예제 #7
0
def update_users():
    rundb = RunDb()

    deltas = {}
    info = {}
    top_month = {}

    clear_stats = True
    if len(sys.argv) > 1:
        print('scan all')
    else:
        deltas = rundb.deltas.find_one()
        if deltas:
            clear_stats = False
        else:
            deltas = {}

    for u in rundb.userdb.get_users():
        username = u['username']
        top_month[username] = {
            'username': username,
            'cpu_hours': 0,
            'games': 0,
            'tests': 0,
            'tests_repo': u.get('tests_repo', ''),
            'last_updated': datetime.min,
            'games_per_hour': 0.0,
        }
        if clear_stats:
            info[username] = top_month[username].copy()
        else:
            info[username] = rundb.userdb.user_cache.find_one(
                {'username': username})
            if not info[username]:
                info[username] = top_month[username].copy()
            else:
                info[username]['games_per_hour'] = 0.0

    for run in rundb.get_unfinished_runs():
        process_run(run, top_month)

    # Step through these in small batches (step size 100) to save RAM
    current = 0
    step_size = 100

    now = datetime.utcnow()
    more_days = True
    while more_days:
        runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
        if len(runs) == 0:
            break
        for run in runs:
            process_run(run, info, deltas)
            if (now - run['start_time']).days < 30:
                process_run(run, top_month)
            elif not clear_stats:
                more_days = False
        current += step_size

    if new_deltas:
        rundb.deltas.remove()
        rundb.deltas.save(new_deltas)

    machines = rundb.get_machines()

    users = build_users(machines, info)
    rundb.userdb.user_cache.remove()
    rundb.userdb.user_cache.insert(users)

    rundb.userdb.top_month.remove()
    rundb.userdb.top_month.insert(build_users(machines, top_month))

    # Delete users that have never been active and old admins group
    idle = {}
    for u in rundb.userdb.get_users():
        update = False
        while 'group:admins' in u['groups']:
            u['groups'].remove('group:admins')
            update = True
        if update:
            rundb.userdb.users.save(u)
        if not 'registration_time' in u \
           or u['registration_time'] < datetime.utcnow() - timedelta(days=28):
            idle[u['username']] = u
    for u in rundb.userdb.user_cache.find():
        if u['username'] in idle:
            del idle[u['username']]
    for u in idle.values():
        # A safe guard against deleting long time users
        if not 'registration_time' in u \
          or u['registration_time'] < datetime.utcnow() - timedelta(days=38):
            print('Warning: Found old user to delete: ' + str(u['_id']))
        else:
            print('Delete: ' + str(u['_id']))
            rundb.userdb.users.remove({'_id': u['_id']})

    print('Successfully updated %d users' % (len(users)))

    # record this update run
    rundb.actiondb.update_stats()
예제 #8
0
def update_users():
    rundb = RunDb()

    deltas = {}
    info = {}
    top_month = {}

    clear_stats = True
    if len(sys.argv) > 1:
        print("scan all")
    else:
        deltas = rundb.deltas.find_one()
        if deltas:
            clear_stats = False
        else:
            deltas = {}

    for u in rundb.userdb.get_users():
        username = u["username"]
        top_month[username] = {
            "username": username,
            "cpu_hours": 0,
            "games": 0,
            "tests": 0,
            "tests_repo": u.get("tests_repo", ""),
            "last_updated": datetime.min,
            "games_per_hour": 0.0,
        }
        if clear_stats:
            info[username] = top_month[username].copy()
        else:
            info[username] = rundb.userdb.user_cache.find_one({"username": username})
            if not info[username]:
                info[username] = top_month[username].copy()
            else:
                info[username]["games_per_hour"] = 0.0

    for run in rundb.get_unfinished_runs():
        try:
            process_run(run, top_month)
        except:
            print("Exception on run: ", run)

    # Step through these in small batches (step size 100) to save RAM
    step_size = 100

    now = datetime.utcnow()
    more_days = True
    last_updated = None
    while more_days:
        q = {"finished": True}
        if last_updated:
            q["last_updated"] = {"$lt": last_updated}
        runs = list(
            rundb.runs.find(q, sort=[("last_updated", DESCENDING)], limit=step_size)
        )
        if len(runs) == 0:
            break
        for run in runs:
            try:
                process_run(run, info, deltas)
            except:
                print("Exception on run: ", run["_id"])
            if (now - run["start_time"]).days < 30:
                try:
                    process_run(run, top_month)
                except:
                    print("Exception on run: ", run["_id"])
            elif not clear_stats:
                more_days = False
        last_updated = runs[-1]["last_updated"]

    if new_deltas:
        new_deltas.update(deltas)
        rundb.deltas.remove()
        rundb.deltas.save(new_deltas)

    machines = rundb.get_machines()

    users = build_users(machines, info)
    rundb.userdb.user_cache.remove()
    rundb.userdb.user_cache.insert(users)
    rundb.userdb.user_cache.create_index("username", unique=True)

    rundb.userdb.top_month.remove()
    rundb.userdb.top_month.insert(build_users(machines, top_month))

    # Delete users that have never been active and old admins group
    idle = {}
    for u in rundb.userdb.get_users():
        update = False
        while "group:admins" in u["groups"]:
            u["groups"].remove("group:admins")
            update = True
        if update:
            rundb.userdb.users.save(u)
        if not "registration_time" in u or u[
            "registration_time"
        ] < datetime.utcnow() - timedelta(days=28):
            idle[u["username"]] = u
    for u in rundb.userdb.user_cache.find():
        if u["username"] in idle:
            del idle[u["username"]]
    for u in idle.values():
        # A safe guard against deleting long time users
        if not "registration_time" in u or u[
            "registration_time"
        ] < datetime.utcnow() - timedelta(days=38):
            print("Warning: Found old user to delete: " + str(u["_id"]))
        else:
            print("Delete: " + str(u["_id"]))
            rundb.userdb.users.remove({"_id": u["_id"]})

    print("Successfully updated %d users" % (len(users)))

    # record this update run
    rundb.actiondb.update_stats()