def update_users():
rundb = RunDb()
info = {}
top_month = {}
for u in rundb.userdb.get_users():
username = u['username']
info[username] = {
'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.datetime.min,
'games_per_hour': 0.0,
}
top_month[username] = info[username].copy()
for run in rundb.get_unfinished_runs():
process_run(run, info)
process_run(run, top_month)
# Step through these 100 at a time to avoid using too much RAM
current = 0
step_size = 100
now = datetime.datetime.utcnow()
while True:
runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
if len(runs) == 0:
break
for run in runs:
process_run(run, info)
if (now - run['start_time']).days < 31:
process_run(run, top_month)
current += step_size
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
print('Successfully updated %d users' % (len(users)))
def update_users():
rundb = RunDb()
info = {}
top_month = {}
for u in rundb.userdb.get_users():
username = u['username']
info[username] = {'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.datetime.min,
'games_per_hour': 0.0,}
top_month[username] = info[username].copy()
for run in rundb.get_unfinished_runs():
process_run(run, info)
process_run(run, top_month)
# Step through these 100 at a time to avoid using too much RAM
current = 0
step_size = 100
now = datetime.datetime.utcnow()
while True:
runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
if len(runs) == 0:
break
for run in runs:
process_run(run, info)
if (now - run['start_time']).days < 31:
process_run(run, top_month)
current += step_size
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
print('Successfully updated %d users' % (len(users)))
conn = MongoClient(os.getenv('FISHTEST_HOST') or 'localhost')
db = conn[db_name]
runs = db['runs']
pgns = db['pgns']
def printout(s):
print(s)
sys.stdout.flush()
printout("\nFetching unfinished runs ...")
start = time.time()
unfinished_runs = rundb.get_unfinished_runs()
end = time.time()
printout(str(end-start) + "s\nFetching machines ...")
start = time.time()
machines = rundb.get_machines()
end = time.time()
printout(str(end-start) + "s\nFetching finished runs ...")
start = time.time()
finished, num_finished = rundb.get_finished_runs(skip=0, limit=50, username='',
success_only=False, ltc_only=False)
end = time.time()
printout(str(end-start) + "s\nRequesting pgn ...")
if (len(finished) == 0):
def update_users():
rundb = RunDb()
deltas = {}
info = {}
top_month = {}
clear_stats = True
if len(sys.argv) > 1:
print('scan all')
else:
deltas = rundb.deltas.find_one()
if deltas:
clear_stats = False
else:
deltas = {}
for u in rundb.userdb.get_users():
username = u['username']
top_month[username] = {
'username': username,
'cpu_hours': 0,
'games': 0,
'tests': 0,
'tests_repo': u.get('tests_repo', ''),
'last_updated': datetime.min,
'games_per_hour': 0.0,
}
if clear_stats:
info[username] = top_month[username].copy()
else:
info[username] = rundb.userdb.user_cache.find_one(
{'username': username})
if not info[username]:
info[username] = top_month[username].copy()
else:
info[username]['games_per_hour'] = 0.0
for run in rundb.get_unfinished_runs():
process_run(run, top_month)
# Step through these in small batches (step size 100) to save RAM
current = 0
step_size = 100
now = datetime.utcnow()
more_days = True
while more_days:
runs = rundb.get_finished_runs(skip=current, limit=step_size)[0]
if len(runs) == 0:
break
for run in runs:
process_run(run, info, deltas)
if (now - run['start_time']).days < 30:
process_run(run, top_month)
elif not clear_stats:
more_days = False
current += step_size
if new_deltas:
rundb.deltas.remove()
rundb.deltas.save(new_deltas)
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
# Delete users that have never been active and old admins group
idle = {}
for u in rundb.userdb.get_users():
update = False
while 'group:admins' in u['groups']:
u['groups'].remove('group:admins')
update = True
if update:
rundb.userdb.users.save(u)
if not 'registration_time' in u \
or u['registration_time'] < datetime.utcnow() - timedelta(days=28):
idle[u['username']] = u
for u in rundb.userdb.user_cache.find():
if u['username'] in idle:
del idle[u['username']]
for u in idle.values():
# A safe guard against deleting long time users
if not 'registration_time' in u \
or u['registration_time'] < datetime.utcnow() - timedelta(days=38):
print('Warning: Found old user to delete: ' + str(u['_id']))
else:
print('Delete: ' + str(u['_id']))
rundb.userdb.users.remove({'_id': u['_id']})
print('Successfully updated %d users' % (len(users)))
# record this update run
rundb.actiondb.update_stats()
def update_users():
rundb = RunDb()
deltas = {}
info = {}
top_month = {}
clear_stats = True
if len(sys.argv) > 1:
print("scan all")
else:
deltas = rundb.deltas.find_one()
if deltas:
clear_stats = False
else:
deltas = {}
for u in rundb.userdb.get_users():
username = u["username"]
top_month[username] = {
"username": username,
"cpu_hours": 0,
"games": 0,
"tests": 0,
"tests_repo": u.get("tests_repo", ""),
"last_updated": datetime.min,
"games_per_hour": 0.0,
}
if clear_stats:
info[username] = top_month[username].copy()
else:
info[username] = rundb.userdb.user_cache.find_one({"username": username})
if not info[username]:
info[username] = top_month[username].copy()
else:
info[username]["games_per_hour"] = 0.0
for run in rundb.get_unfinished_runs():
try:
process_run(run, top_month)
except:
print("Exception on run: ", run)
# Step through these in small batches (step size 100) to save RAM
step_size = 100
now = datetime.utcnow()
more_days = True
last_updated = None
while more_days:
q = {"finished": True}
if last_updated:
q["last_updated"] = {"$lt": last_updated}
runs = list(
rundb.runs.find(q, sort=[("last_updated", DESCENDING)], limit=step_size)
)
if len(runs) == 0:
break
for run in runs:
try:
process_run(run, info, deltas)
except:
print("Exception on run: ", run["_id"])
if (now - run["start_time"]).days < 30:
try:
process_run(run, top_month)
except:
print("Exception on run: ", run["_id"])
elif not clear_stats:
more_days = False
last_updated = runs[-1]["last_updated"]
if new_deltas:
new_deltas.update(deltas)
rundb.deltas.remove()
rundb.deltas.save(new_deltas)
machines = rundb.get_machines()
users = build_users(machines, info)
rundb.userdb.user_cache.remove()
rundb.userdb.user_cache.insert(users)
rundb.userdb.user_cache.create_index("username", unique=True)
rundb.userdb.top_month.remove()
rundb.userdb.top_month.insert(build_users(machines, top_month))
# Delete users that have never been active and old admins group
idle = {}
for u in rundb.userdb.get_users():
update = False
while "group:admins" in u["groups"]:
u["groups"].remove("group:admins")
update = True
if update:
rundb.userdb.users.save(u)
if not "registration_time" in u or u[
"registration_time"
] < datetime.utcnow() - timedelta(days=28):
idle[u["username"]] = u
for u in rundb.userdb.user_cache.find():
if u["username"] in idle:
del idle[u["username"]]
for u in idle.values():
# A safe guard against deleting long time users
if not "registration_time" in u or u[
"registration_time"
] < datetime.utcnow() - timedelta(days=38):
print("Warning: Found old user to delete: " + str(u["_id"]))
else:
print("Delete: " + str(u["_id"]))
rundb.userdb.users.remove({"_id": u["_id"]})
print("Successfully updated %d users" % (len(users)))
# record this update run
rundb.actiondb.update_stats()
def find_run(arg="username", value="travis"):
rundb = RunDb(db_name="fishtest_tests")
for run in rundb.get_unfinished_runs():
if run["args"][arg] == value:
return run
return None
def find_run(arg='username', value='travis'):
rundb = RunDb()
for run in rundb.get_unfinished_runs():
if run['args'][arg] == value:
return run
return None
def find_run(arg='username', value='travis'):
rundb= RunDb()
for r in rundb.get_unfinished_runs():
if r['args'][arg] == value:
return r
return None
