def update_users():
rundb = RunDb()
info = {}
top_month = {}
for u in rundb.userdb.get_users():
username = u['username']
info[username] = {
'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.datetime.min,
'games_per_hour': 0.0,
}
top_month[username] = info[username].copy()
for run in rundb.get_unfinished_runs():
process_run(run, info)
process_run(run, top_month)
# Step through these 100 at a time to avoid using too much RAM
current = 0
step_size = 100
now = datetime.datetime.utcnow()
while True:
runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
if len(runs) == 0:
break
for run in runs:
process_run(run, info)
if (now - run['start_time']).days < 31:
process_run(run, top_month)
current += step_size
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
print('Successfully updated %d users' % (len(users)))
def update_users():
rundb = RunDb()
info = {}
top_month = {}
for u in rundb.userdb.get_users():
username = u['username']
info[username] = {'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.datetime.min,
'games_per_hour': 0.0,}
top_month[username] = info[username].copy()
for run in rundb.get_unfinished_runs():
process_run(run, info)
process_run(run, top_month)
# Step through these 100 at a time to avoid using too much RAM
current = 0
step_size = 100
now = datetime.datetime.utcnow()
while True:
runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
if len(runs) == 0:
break
for run in runs:
process_run(run, info)
if (now - run['start_time']).days < 31:
process_run(run, top_month)
current += step_size
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
print('Successfully updated %d users' % (len(users)))
print(s)
sys.stdout.flush()
printout("\nFetching unfinished runs ...")
start = time.time()
unfinished_runs = rundb.get_unfinished_runs()
end = time.time()
printout(str(end-start) + "s\nFetching machines ...")
start = time.time()
machines = rundb.get_machines()
end = time.time()
printout(str(end-start) + "s\nFetching finished runs ...")
start = time.time()
finished, num_finished = rundb.get_finished_runs(skip=0, limit=50, username='',
success_only=False, ltc_only=False)
end = time.time()
printout(str(end-start) + "s\nRequesting pgn ...")
if (len(finished) == 0):
finished.append({'_id':'abc'})
start = time.time()
pgn = rundb.get_pgn(str(finished[0]['_id']) + ".pgn")
end = time.time()
printout(str(end-start) + "s\n")
def update_users():
rundb = RunDb()
deltas = {}
info = {}
top_month = {}
clear_stats = True
if len(sys.argv) > 1:
print('scan all')
else:
deltas = rundb.deltas.find_one()
if deltas:
clear_stats = False
else:
deltas = {}
for u in rundb.userdb.get_users():
username = u['username']
top_month[username] = {
'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.min,
'games_per_hour': 0.0,
}
if clear_stats:
info[username] = top_month[username].copy()
else:
info[username] = rundb.userdb.user_cache.find_one(
{'username': username})
if not info[username]:
info[username] = top_month[username].copy()
else:
info[username]['games_per_hour'] = 0.0
for run in rundb.get_unfinished_runs():
process_run(run, top_month)
# Step through these in small batches (step size 100) to save RAM
current = 0
step_size = 100
now = datetime.utcnow()
more_days = True
while more_days:
runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
if len(runs) == 0:
break
for run in runs:
process_run(run, info, deltas)
if (now - run['start_time']).days < 30:
process_run(run, top_month)
elif not clear_stats:
more_days = False
current += step_size
if new_deltas:
rundb.deltas.remove()
rundb.deltas.save(new_deltas)
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
# Delete users that have never been active and old admins group
idle = {}
for u in rundb.userdb.get_users():
update = False
while 'group:admins' in u['groups']:
u['groups'].remove('group:admins')
update = True
if update:
rundb.userdb.users.save(u)
if not 'registration_time' in u \
or u['registration_time'] < datetime.utcnow() - timedelta(days=28):
idle[u['username']] = u
for u in rundb.userdb.user_cache.find():
if u['username'] in idle:
del idle[u['username']]
for u in idle.values():
# A safe guard against deleting long time users
if not 'registration_time' in u \
or u['registration_time'] < datetime.utcnow() - timedelta(days=38):
print('Warning: Found old user to delete: ' + str(u['_id']))
else:
print('Delete: ' + str(u['_id']))
rundb.userdb.users.remove({'_id': u['_id']})
print('Successfully updated %d users' % (len(users)))
# record this update run
rundb.actiondb.update_stats()
start = time.time()
c = rundb.get_unfinished_runs()
end = time.time()
print("{} rows {:1.4f}".format(qlen(c), end - start) +
"s\nFetching machines ...")
start = time.time()
c = rundb.get_machines()
end = time.time()
print("{} rows {:1.4f}".format(qlen(c), end - start) +
"s\nFetching finished runs ...")
start = time.time()
c, n = rundb.get_finished_runs(skip=0,
limit=50,
username="",
success_only=False,
ltc_only=False)
end = time.time()
print("{} rows {:1.4f}".format(qlen(c), end - start) +
"s\nFetching finished runs (vdv) ...")
start = time.time()
c, n = rundb.get_finished_runs(skip=0,
limit=50,
username="******",
success_only=False,
ltc_only=False)
end = time.time()
print("{} rows {:1.4f}".format(qlen(c), end - start) + "s\nRequesting pgn ...")
