def trim(test=False):
"""
Implementation of a FIFO log window by trimming logs from X days ago from MongoDB
by deleting old collections
"""
# for every log type, delete the collection from a certain amount of days ago (in config)
for log_type in get_log_types():
try:
oldest_date_to_keep = int(get_date_x_days_ago(int(get_config(log_type, 'window'))))
db = connect_db(log_type, local=test, master=True)
# grab every collection and if it's older than the log window, drop it
for collection in db.collection_names():
collection_date = collection[-8:]
try:
if int(collection_date) < int(oldest_date_to_keep):
db.drop_collection(collection)
except:
continue
except Exception as e:
print str(e)
continue
def trim(test=False):
"""
Implementation of a FIFO log window by trimming logs from X days ago from MongoDB
by deleting old collections
"""
# for every log type, delete the collection from a certain amount of days ago (in config)
for log_type in get_log_types():
try:
oldest_date_to_keep = int(
get_date_x_days_ago(int(get_config(log_type, 'window'))))
db = connect_db(log_type, local=test, master=True)
# grab every collection and if it's older than the log window, drop it
for collection in db.collection_names():
collection_date = collection[-8:]
try:
if int(collection_date) < int(oldest_date_to_keep):
db.drop_collection(collection)
except:
continue
except Exception as e:
print str(e)
continue
def get_stats():
for log_type in get_log_types():
db = connect_db(log_type)
log_stats = db.command("dbstats")
stats_coll = connect_collection('tools', 'db_stats')
stats_object = stats_coll.update({'db':log_type}, log_stats, upsert=True)
def tests(self):
for case in self.test_list:
case["dumper"].dump(test=True)
names = connect_db(case["logtype"], local=True).collection_names()
coll = None
names.remove("system.indexes")
try:
coll = connect_collection(case["logtype"], names[0], local=True)
self.assertEqual(coll.count() % 5, 0)
for test in self.test_keys:
assert coll.distinct(test) is not 0
except Exception as e:
print case["logtype"] + " had an error"
print str(e)
pass
def print_last_inserts():
""" return time of last insert for each logtype """
for log_type in get_log_types():
newest_coll_name = connect_db(log_type, local=False).collection_names()[-1]
coll = connect_collection(log_type, newest_coll_name, local=False)
for document in coll.find().sort([('time', DESCENDING)]).limit(1):
line = document['line'].split(' ')
try:
line.remove('')
except:
pass
print log_type + '\t\t' + str(line[0:3])
def setUp(self):
self.dbs = ["cca", "csacs", "dhcp", "greylist", "header_reject", "named", "nat", "pat", "resnet", "vpn"]
self.dumpers = [
cca_dump,
csacs_dump,
dhcp_dump,
greylist_dump,
header_reject_dump,
named_dump,
nat_dump,
pat_dump,
resnet_dump,
vpn_dump,
]
self.queries = [
{"username": "******"},
{"mac": "aabbccddeeff"},
{"recipient": "*****@*****.**"},
{"from_addr": "*****@*****.**"},
{"zone": "osupachyderm.org"},
{"time": 1319029072},
{"port": "60940"},
{"username": "******"},
]
self.test_keys = ["username", "mac", "recipient", "from_addr", "zone", "time", "port", "ip"]
self.test_list = [
{"dumper": cca_dump, "logtype": "cca"},
{"dumper": csacs_dump, "logtype": "csacs"},
{"dumper": dhcp_dump, "logtype": "dhcp"},
{"dumper": greylist_dump, "logtype": "greylist"},
{"dumper": header_reject_dump, "logtype": "header_reject"},
{"dumper": named_dump, "logtype": "named"},
{"dumper": nat_dump, "logtype": "nat"},
{"dumper": pat_dump, "logtype": "pat"},
{"dumper": resnet_dump, "logtype": "resnet"},
{"dumper": vpn_dump, "logtype": "vpn"},
]
for case in self.test_list:
db = connect_db(case["logtype"], local=True)
names = db.collection_names()
try:
names.remove("system.indexes")
except:
pass
def compact(test=False):
"""
Compact (defrag/free) every collection
"""
for log_type in get_log_types():
try:
db = connect_db(log_type, local=test, master=True)
for coll_name in db.collection_names():
coll = connect_collection(log_type, coll_name)
try:
print "compacting " + coll_name
db.eval("db.runCommand({ compact : '%s'})" % (coll_name))
except Exception as e:
print str(e)
continue
except Exception as e:
print str(e)
continue
Prints statistics summarizing the frequency of each key
in a collection of a Mongo database. Helpful as a
diagnostic tool.
"""
<<<<<<< Updated upstream
# count all documents in this collection
total = collection.count()
#f or log_type in get_log_types():
=======
# for log_type in get_log_types():
>>>>>>> Stashed changes
for log_type in ['aruba']:
# find all collections in the database
database = connect_db(log_type, local=True)
collnames = database.collection_names()
try:
collnames.remove('system.indexes')
collnames.remove('system.users')
except:
pass
# loop over collections names
for collname in collnames:
collection = connect_collection(log_type, collname, local=True)
<<<<<<< Updated upstream
>>>>>>> Stashed changes
=======
>>>>>>> Stashed changes
