def update_users(): rundb = RunDb() info = {} top_month = {} for u in rundb.userdb.get_users(): username = u['username'] info[username] = { 'username': username, 'cpu_hours': 0, 'games': 0, 'tests': 0, 'tests_repo': u.get('tests_repo', ''), 'last_updated': datetime.datetime.min, 'games_per_hour': 0.0, } top_month[username] = info[username].copy() for run in rundb.get_unfinished_runs(): process_run(run, info) process_run(run, top_month) # Step through these 100 at a time to avoid using too much RAM current = 0 step_size = 100 now = datetime.datetime.utcnow() while True: runs = rundb.get_finished_runs(skip=current, limit=step_size)[0] if len(runs) == 0: break for run in runs: process_run(run, info) if (now - run['start_time']).days < 31: process_run(run, top_month) current += step_size machines = rundb.get_machines() users = build_users(machines, info) rundb.userdb.user_cache.remove() rundb.userdb.user_cache.insert(users) rundb.userdb.top_month.remove() rundb.userdb.top_month.insert(build_users(machines, top_month)) print('Successfully updated %d users' % (len(users)))
def update_users(): rundb = RunDb() info = {} top_month = {} for u in rundb.userdb.get_users(): username = u['username'] info[username] = {'username': username, 'cpu_hours': 0, 'games': 0, 'tests': 0, 'tests_repo': u.get('tests_repo', ''), 'last_updated': datetime.datetime.min, 'games_per_hour': 0.0,} top_month[username] = info[username].copy() for run in rundb.get_unfinished_runs(): process_run(run, info) process_run(run, top_month) # Step through these 100 at a time to avoid using too much RAM current = 0 step_size = 100 now = datetime.datetime.utcnow() while True: runs = rundb.get_finished_runs(skip=current, limit=step_size)[0] if len(runs) == 0: break for run in runs: process_run(run, info) if (now - run['start_time']).days < 31: process_run(run, top_month) current += step_size machines = rundb.get_machines() users = build_users(machines, info) rundb.userdb.user_cache.remove() rundb.userdb.user_cache.insert(users) rundb.userdb.top_month.remove() rundb.userdb.top_month.insert(build_users(machines, top_month)) print('Successfully updated %d users' % (len(users)))
conn = MongoClient(os.getenv('FISHTEST_HOST') or 'localhost') db = conn[db_name] runs = db['runs'] pgns = db['pgns'] def printout(s): print(s) sys.stdout.flush() printout("\nFetching unfinished runs ...") start = time.time() unfinished_runs = rundb.get_unfinished_runs() end = time.time() printout(str(end-start) + "s\nFetching machines ...") start = time.time() machines = rundb.get_machines() end = time.time() printout(str(end-start) + "s\nFetching finished runs ...") start = time.time() finished, num_finished = rundb.get_finished_runs(skip=0, limit=50, username='', success_only=False, ltc_only=False) end = time.time() printout(str(end-start) + "s\nRequesting pgn ...") if (len(finished) == 0):
def update_users(): rundb = RunDb() deltas = {} info = {} top_month = {} clear_stats = True if len(sys.argv) > 1: print('scan all') else: deltas = rundb.deltas.find_one() if deltas: clear_stats = False else: deltas = {} for u in rundb.userdb.get_users(): username = u['username'] top_month[username] = { 'username': username, 'cpu_hours': 0, 'games': 0, 'tests': 0, 'tests_repo': u.get('tests_repo', ''), 'last_updated': datetime.min, 'games_per_hour': 0.0, } if clear_stats: info[username] = top_month[username].copy() else: info[username] = rundb.userdb.user_cache.find_one( {'username': username}) if not info[username]: info[username] = top_month[username].copy() else: info[username]['games_per_hour'] = 0.0 for run in rundb.get_unfinished_runs(): process_run(run, top_month) # Step through these in small batches (step size 100) to save RAM current = 0 step_size = 100 now = datetime.utcnow() more_days = True while more_days: runs = rundb.get_finished_runs(skip=current, limit=step_size)[0] if len(runs) == 0: break for run in runs: process_run(run, info, deltas) if (now - run['start_time']).days < 30: process_run(run, top_month) elif not clear_stats: more_days = False current += step_size if new_deltas: rundb.deltas.remove() rundb.deltas.save(new_deltas) machines = rundb.get_machines() users = build_users(machines, info) rundb.userdb.user_cache.remove() rundb.userdb.user_cache.insert(users) rundb.userdb.top_month.remove() rundb.userdb.top_month.insert(build_users(machines, top_month)) # Delete users that have never been active and old admins group idle = {} for u in rundb.userdb.get_users(): update = False while 'group:admins' in u['groups']: u['groups'].remove('group:admins') update = True if update: rundb.userdb.users.save(u) if not 'registration_time' in u \ or u['registration_time'] < datetime.utcnow() - timedelta(days=28): idle[u['username']] = u for u in rundb.userdb.user_cache.find(): if u['username'] in idle: del idle[u['username']] for u in idle.values(): # A safe guard against deleting long time users if not 'registration_time' in u \ or u['registration_time'] < datetime.utcnow() - timedelta(days=38): print('Warning: Found old user to delete: ' + str(u['_id'])) else: print('Delete: ' + str(u['_id'])) rundb.userdb.users.remove({'_id': u['_id']}) print('Successfully updated %d users' % (len(users))) # record this update run rundb.actiondb.update_stats()
def update_users(): rundb = RunDb() deltas = {} info = {} top_month = {} clear_stats = True if len(sys.argv) > 1: print("scan all") else: deltas = rundb.deltas.find_one() if deltas: clear_stats = False else: deltas = {} for u in rundb.userdb.get_users(): username = u["username"] top_month[username] = { "username": username, "cpu_hours": 0, "games": 0, "tests": 0, "tests_repo": u.get("tests_repo", ""), "last_updated": datetime.min, "games_per_hour": 0.0, } if clear_stats: info[username] = top_month[username].copy() else: info[username] = rundb.userdb.user_cache.find_one({"username": username}) if not info[username]: info[username] = top_month[username].copy() else: info[username]["games_per_hour"] = 0.0 for run in rundb.get_unfinished_runs(): try: process_run(run, top_month) except: print("Exception on run: ", run) # Step through these in small batches (step size 100) to save RAM step_size = 100 now = datetime.utcnow() more_days = True last_updated = None while more_days: q = {"finished": True} if last_updated: q["last_updated"] = {"$lt": last_updated} runs = list( rundb.runs.find(q, sort=[("last_updated", DESCENDING)], limit=step_size) ) if len(runs) == 0: break for run in runs: try: process_run(run, info, deltas) except: print("Exception on run: ", run["_id"]) if (now - run["start_time"]).days < 30: try: process_run(run, top_month) except: print("Exception on run: ", run["_id"]) elif not clear_stats: more_days = False last_updated = runs[-1]["last_updated"] if new_deltas: new_deltas.update(deltas) rundb.deltas.remove() rundb.deltas.save(new_deltas) machines = rundb.get_machines() users = build_users(machines, info) rundb.userdb.user_cache.remove() rundb.userdb.user_cache.insert(users) rundb.userdb.user_cache.create_index("username", unique=True) rundb.userdb.top_month.remove() rundb.userdb.top_month.insert(build_users(machines, top_month)) # Delete users that have never been active and old admins group idle = {} for u in rundb.userdb.get_users(): update = False while "group:admins" in u["groups"]: u["groups"].remove("group:admins") update = True if update: rundb.userdb.users.save(u) if not "registration_time" in u or u[ "registration_time" ] < datetime.utcnow() - timedelta(days=28): idle[u["username"]] = u for u in rundb.userdb.user_cache.find(): if u["username"] in idle: del idle[u["username"]] for u in idle.values(): # A safe guard against deleting long time users if not "registration_time" in u or u[ "registration_time" ] < datetime.utcnow() - timedelta(days=38): print("Warning: Found old user to delete: " + str(u["_id"])) else: print("Delete: " + str(u["_id"])) rundb.userdb.users.remove({"_id": u["_id"]}) print("Successfully updated %d users" % (len(users))) # record this update run rundb.actiondb.update_stats()
def find_run(arg="username", value="travis"): rundb = RunDb(db_name="fishtest_tests") for run in rundb.get_unfinished_runs(): if run["args"][arg] == value: return run return None
def find_run(arg='username', value='travis'): rundb = RunDb() for run in rundb.get_unfinished_runs(): if run['args'][arg] == value: return run return None
def find_run(arg='username', value='travis'): rundb= RunDb() for r in rundb.get_unfinished_runs(): if r['args'][arg] == value: return r return None