def drop(user, password): """ Drop every database and re-add net user account (CAREFUL BEFORE RUNNING) """ connection = Connection(get_config('database', 'master_host'), int(get_config('database', 'port'))) db = database.Database(connection, 'admin') db.authenticate(user, password) for log_type in get_log_types(): try: db = database.Database(connection, log_type) print "dropping " + log_type connection.drop_database(log_type) # re-add net user account db.add_user(get_config('database', 'user'), get_config('database', 'password')) except Exception as e: print str(e) continue
def trim(test=False): """ Implementation of a FIFO log window by trimming logs from X days ago from MongoDB by deleting old collections """ # for every log type, delete the collection from a certain amount of days ago (in config) for log_type in get_log_types(): try: oldest_date_to_keep = int(get_date_x_days_ago(int(get_config(log_type, 'window')))) db = connect_db(log_type, local=test, master=True) # grab every collection and if it's older than the log window, drop it for collection in db.collection_names(): collection_date = collection[-8:] try: if int(collection_date) < int(oldest_date_to_keep): db.drop_collection(collection) except: continue except Exception as e: print str(e) continue
def trim(test=False): """ Implementation of a FIFO log window by trimming logs from X days ago from MongoDB by deleting old collections """ # for every log type, delete the collection from a certain amount of days ago (in config) for log_type in get_log_types(): try: oldest_date_to_keep = int( get_date_x_days_ago(int(get_config(log_type, 'window')))) db = connect_db(log_type, local=test, master=True) # grab every collection and if it's older than the log window, drop it for collection in db.collection_names(): collection_date = collection[-8:] try: if int(collection_date) < int(oldest_date_to_keep): db.drop_collection(collection) except: continue except Exception as e: print str(e) continue
def get_stats(): for log_type in get_log_types(): db = connect_db(log_type) log_stats = db.command("dbstats") stats_coll = connect_collection('tools', 'db_stats') stats_object = stats_coll.update({'db':log_type}, log_stats, upsert=True)
def print_last_inserts(): """ return time of last insert for each logtype """ for log_type in get_log_types(): newest_coll_name = connect_db(log_type, local=False).collection_names()[-1] coll = connect_collection(log_type, newest_coll_name, local=False) for document in coll.find().sort([('time', DESCENDING)]).limit(1): line = document['line'].split(' ') try: line.remove('') except: pass print log_type + '\t\t' + str(line[0:3])
def compact(test=False): """ Compact (defrag/free) every collection """ for log_type in get_log_types(): try: db = connect_db(log_type, local=test, master=True) for coll_name in db.collection_names(): coll = connect_collection(log_type, coll_name) try: print "compacting " + coll_name db.eval("db.runCommand({ compact : '%s'})" % (coll_name)) except Exception as e: print str(e) continue except Exception as e: print str(e) continue