def drop(user, password):
"""
Drop every database and re-add net user account (CAREFUL BEFORE RUNNING)
"""
connection = Connection(get_config('database', 'master_host'),
int(get_config('database', 'port')))
db = database.Database(connection, 'admin')
db.authenticate(user, password)
for log_type in get_log_types():
try:
db = database.Database(connection, log_type)
print "dropping " + log_type
connection.drop_database(log_type)
# re-add net user account
db.add_user(get_config('database', 'user'),
get_config('database', 'password'))
except Exception as e:
print str(e)
continue
def trim(test=False):
"""
Implementation of a FIFO log window by trimming logs from X days ago from MongoDB
by deleting old collections
"""
# for every log type, delete the collection from a certain amount of days ago (in config)
for log_type in get_log_types():
try:
oldest_date_to_keep = int(get_date_x_days_ago(int(get_config(log_type, 'window'))))
db = connect_db(log_type, local=test, master=True)
# grab every collection and if it's older than the log window, drop it
for collection in db.collection_names():
collection_date = collection[-8:]
try:
if int(collection_date) < int(oldest_date_to_keep):
db.drop_collection(collection)
except:
continue
except Exception as e:
print str(e)
continue
def trim(test=False):
"""
Implementation of a FIFO log window by trimming logs from X days ago from MongoDB
by deleting old collections
"""
# for every log type, delete the collection from a certain amount of days ago (in config)
for log_type in get_log_types():
try:
oldest_date_to_keep = int(
get_date_x_days_ago(int(get_config(log_type, 'window'))))
db = connect_db(log_type, local=test, master=True)
# grab every collection and if it's older than the log window, drop it
for collection in db.collection_names():
collection_date = collection[-8:]
try:
if int(collection_date) < int(oldest_date_to_keep):
db.drop_collection(collection)
except:
continue
except Exception as e:
print str(e)
continue
def get_stats():
for log_type in get_log_types():
db = connect_db(log_type)
log_stats = db.command("dbstats")
stats_coll = connect_collection('tools', 'db_stats')
stats_object = stats_coll.update({'db':log_type}, log_stats, upsert=True)
def print_last_inserts():
""" return time of last insert for each logtype """
for log_type in get_log_types():
newest_coll_name = connect_db(log_type, local=False).collection_names()[-1]
coll = connect_collection(log_type, newest_coll_name, local=False)
for document in coll.find().sort([('time', DESCENDING)]).limit(1):
line = document['line'].split(' ')
try:
line.remove('')
except:
pass
print log_type + '\t\t' + str(line[0:3])
def compact(test=False):
"""
Compact (defrag/free) every collection
"""
for log_type in get_log_types():
try:
db = connect_db(log_type, local=test, master=True)
for coll_name in db.collection_names():
coll = connect_collection(log_type, coll_name)
try:
print "compacting " + coll_name
db.eval("db.runCommand({ compact : '%s'})" % (coll_name))
except Exception as e:
print str(e)
continue
except Exception as e:
print str(e)
continue
def drop(user, password):
"""
Drop every database and re-add net user account (CAREFUL BEFORE RUNNING)
"""
connection = Connection(get_config('database', 'master_host'), int(get_config('database', 'port')))
db = database.Database(connection, 'admin')
db.authenticate(user, password)
for log_type in get_log_types():
try:
db = database.Database(connection, log_type)
print "dropping " + log_type
connection.drop_database(log_type)
# re-add net user account
db.add_user(get_config('database', 'user'), get_config('database', 'password'))
except Exception as e:
print str(e)
continue
