class Mimosa(Cmd):
dbconn = None
prompt="Mimosa> "
def get_db(self):
try:
if self.dbconn == None:
self.dbconn = Connection()
self.dbconn.server_info()
return self.dbconn['mimosa']
except Exception, e:
if e.__module__ == "pymongo.errors":
self.dbconn = None
print(self.colorize("Database error. Is MongoDB running?", "red")),
print self.colorize("Cannot continue.", "red")
return None
class PlumageSessionThread(MintDaemonThread):
def __init__(self, app, server_host, server_port, cls):
super(PlumageSessionThread, self).__init__(app)
self.cls = cls
self.server_host = server_host
self.server_port = server_port
try:
self.connection = Connection(server_host, server_port)
except AutoReconnect:
self.connection = None
log.info("%s could not connect to mongo server at %s:%s, "\
"autoreconnect is on" % (self.__class__.__name__,
server_host,
server_port))
self.database = None
self.collection = None
def _init(self):
if self.database is None:
self.database = Database(self.connection, self.cls.database)
if self.collection is None:
self.collection = Collection(self.database, self.cls.collection)
def _check_connection(self):
# Loop waiting for the server if it's not there. Autoreconnect
# will hook us up...
while not self.stop_requested:
try:
if self.connection is None:
self.connection = Connection(self.server_host,
self.server_port)
else:
self.connection.server_info()
break
except AutoReconnect, e:
log.info("%s waiting for server, %s, sleeping..."\
% (self.__class__.__name__, str(e)))
for i in range(10):
sleep(1)
if self.stop_requested:
break
class Mimosa(Cmd):
dbconn = None
prompt = "Mimosa> "
def get_db(self):
try:
if self.dbconn == None:
self.dbconn = Connection()
self.dbconn.server_info()
return self.dbconn['mimosa']
except Exception, e:
if e.__module__ == "pymongo.errors":
self.dbconn = None
print(
self.colorize("Database error. Is MongoDB running?",
"red")),
print self.colorize("Cannot continue.", "red")
return None
def makeDBConnection(dbport):
global _C
if not _C:
logging.info("establishing db connection at port %s ..." % dbport)
import pymongo
logging.info("using pymongo version %s" % pymongo.version)
if pymongo.version_tuple[0] < 3:
from pymongo import Connection
_C = Connection(port=dbport)
else:
from pymongo.mongo_client import MongoClient
_C = MongoClient(port=dbport)
mongo_info = _C.server_info()
logging.info("mongodb version: %s" % mongo_info["version"])
def makeDBConnection(dbport):
global _C
if not _C:
logging.info("establishing db connection at port %s ..." % dbport)
import pymongo
logging.info("using pymongo version %s" % pymongo.version)
if pymongo.version_tuple[0] < 3:
from pymongo import Connection
_C = Connection(port=dbport)
else:
from pymongo.mongo_client import MongoClient
_C = MongoClient(port=dbport)
mongo_info = _C.server_info()
logging.info("mongodb version: %s" % mongo_info["version"])
def view(server_oid):
server = g.db['mangoadmin']['servers'].find_one({
'_id': ObjectId(server_oid)
})
if not server:
flash('Server %s not found' % server_oid, 'error')
return redirect('/servers')
connection = Connection(host=server['address'], port=int(server['port']))
server_info = connection.server_info()
databases = {
database: {
collection : {
'count': connection[database][collection].count(),
'index_count': len(connection[database][collection].index_information().keys()),
} for collection in connection[database].collection_names() if not collection == 'system.indexes'
}
for database in connection.database_names()
}
return render_template(
'server_view.html',
server=server,
server_info=server_info,
databases=databases
)
class Instance(object):
def __init__(self, main_config, node, name, replSet=None, logger_level=logging.INFO):
self.logger = logging.getLogger('instance-%s' % name)
self.logger.setLevel(logger_level)
self.main_config = main_config
self.name = name
self.node = node
mongo_dbin = node.config['mongo_dbin']
mongo_dbpath = "%s/%s/" % (node.config['mongo_dbpath'], name)
mongo_logpath = "%s/%s.log" % (node.config['mongo_logpath'], name)
mongo_pidpath = "%s/%s.pid" % (node.config['mongo_pidpath'], name)
default_config = {
'flags': None,
'port': 27017,
'mongo_dbin': mongo_dbin,
'mongo_logpath': mongo_logpath,
'mongo_pidpath': mongo_pidpath,
'mongo_dbpath': mongo_dbpath,
'managed': True,
'replset': None
}
self.config = get_config(main_config, 'instance-%s' % name, default_config)
#self.logger.debug('Config: %s' % self.config)
# Set by replicaSet
self.replSet = replSet
self.logger.debug('Config replset: %s' % self.config['replset'])
self.pid = None
self.connected = False
self.mclient = None
self.minfo = None
self.me = None
self.isMaster = False
self.isPrimary = False
self.isSecondary = False
def make_cmd(self):
cmd = self.config['mongo_dbin']
cmd += " --logpath %s" % self.config['mongo_logpath']
cmd += " --dbpath %s" % self.config['mongo_dbpath']
cmd += " --pidfilepath %s" % self.config['mongo_pidpath']
cmd += " --port %s" % self.config['port']
if self.replSet:
cmd += " --replSet %s " % self.replSet.name
if self.config['flags']:
cmd += " %s " % self.config['flags']
cmd += "--logappend --fork"
self.logger.debug('Command line: %s' % cmd)
return cmd
def get_pid(self):
#if self.pid:
# return self.pid
if not self.node.file_exist(self.config['mongo_pidpath']):
return None
pid = self.node.exec_command("cat %s | tail -n 1" % self.config['mongo_pidpath'])
self.logger.debug("Pid: %s" % pid)
try:
pid = int(pid)
return pid
except:
return None
def ping(self):
return tcp_ping(self.node.config["host"], self.config["port"])
def get_state(self):
state = True
self.states = {}
self.pid = self.get_pid()
self.states['pid'] = self.pid
self.states['pid_run'] = False
self.states['tcp_ping'] = self.ping()
self.states['lock'] = False
try:
lock = self.node.exec_command("cat %s/mongod.lock | tail -n 1" % self.config['mongo_dbpath'])
if lock and int(lock) == self.pid:
self.states['lock'] = True
except:
pass
self.states['running'] = self.check_running()
if self.pid:
self.states['pid_run'] = self.node.file_exist("/proc/%s/cmdline" % self.pid)
if self.states['lock']:
self.connect()
self.states['replSet'] = False
if self.minfo and self.replSet:
self.states['replSet'] = self.minfo["isMaster"].get("setName", None) == self.replSet.name
for key in self.states:
state &= bool(self.states[key])
self.state = state
return state
def check_running(self):
self.pid = self.get_pid()
state = self.ping()
if state and not self.connected:
state &= bool(self.connect())
return state
def start(self):
print("Start " + str_blue(self.name) + " on " + str_blue(self.node.name) + ":")
if not self.config['managed']:
print(" + Not managed")
return True
self.node.exec_command("mkdir -p %s" % self.config['mongo_dbpath'])
state = self.check_running()
if state:
print(" + Already started")
return True
if not self.node.file_exist(self.config['mongo_dbin']):
str_state(False, nok=" + Impossible to find mongod binary")
return False
cmd = self.make_cmd()
self.node.exec_command(cmd)
for i in range(0, 10):
state = self.check_running()
if state:
print(" + " + str_state(True, ok="Done"))
return True
time.sleep(1)
print(" + " + str_state(False, nok="Fail"))
print("See '%s' for more informations ..." % self.config['mongo_logpath'])
return False
def stop(self):
print("Stop " + str_blue(self.name) + " on " + str_blue(self.node.name) + ":")
if not self.config['managed']:
print(" + Not managed")
return True
state = self.check_running()
if not state:
print(" + Already stoped")
return True
pid = self.get_pid()
if pid:
self.node.exec_command("kill %s" % pid)
else:
self.logger.warning("Invalid Pid")
return
for i in range(0, 10):
state = self.check_running()
if not state:
print(" + " + str_state(True, ok="Done"))
return True
time.sleep(1)
print(" + " + str_state(False, nok="Fail"))
return False
def reset(self, prompt=True):
print("Reset " + str_blue(self.name) + " on " + str_blue(self.node.name) + ":")
do = True
if prompt:
do = False
try:
choice = raw_input('Warning, this operation erase all your DB data, are you sure ? (yes/no): ')
except:
print('')
sys.exit(1)
if choice == "yes":
do = True
if not do:
return
self.node.exec_command("rm -Rf %s" % self.config['mongo_dbpath'])
self.node.exec_command("rm %s" % self.config['mongo_pidpath'])
self.node.exec_command("rm %s" % self.config['mongo_logpath'])
self.node.exec_command("rm %s.*" % self.config['mongo_logpath'])
print(" + " + str_state(True, ok="Done"))
return True
def connect(self):
if not self.connected:
try:
self.mclient = Connection(self.node.config['host'], self.config['port'], slaveOk=True)
self.minfo = self.mclient.server_info()
self.connected = True
self.minfo['isMaster'] = self.mclient["admin"].command("isMaster")
self.logger.debug("info: %s" % self.minfo)
self.logger.debug("isMaster: %s" % self.minfo['isMaster'])
self.me = self.minfo['isMaster'].get("me", None)
self.setName = self.minfo['isMaster'].get("setName", None)
self.isMaster = self.minfo['isMaster'].get("ismaster", False)
self.isArbiter = self.me in self.minfo['isMaster'].get("arbiters", [])
self.isPrimary = self.me == self.minfo['isMaster'].get("primary", 'noOne')
self.isSecondary = self.minfo['isMaster'].get("secondary", False)
return self.minfo
except Exception as err:
self.logger.error(err)
return None
def disconnect(self):
if self.connected:
self.mclient.disconnect()
self.connected = False
def replSet_addMember(self, member):
print("Add member " + str_blue(member.name) + " on " + str_blue(self.replSet.name) )
#rs.add("wpain-laptop:2002")
#self.mclient.admin.command("rs.add('wpain-laptop:20000')")
def status(self):
state = self.get_state()
print("State of " + str_blue(self.name) + " on " + str_blue(self.node.name) + ": " + str_state(state))
cprint(" + Node:", str_state(self.node.get_state()) )
cprint(" + Managed:", str_state(self.config['managed'], ok="Yes", nok="No") )
cprint(" + Sarted:", str_state(self.states['running'], ok="Yes", nok="No") )
cprint(" + Port:", self.config['port'] )
cprint(" + TCP Ping:", str_state(self.states['tcp_ping']) )
cprint(" + Lock:", str_state(self.states['lock']) )
if self.connected:
cprint(" + replset:", str_state(self.states['replSet'], ok="Yes", nok="No"))
cprint(" + PID:", str_state(self.pid, ok=self.pid) )
cprint(" + Mongod:", "%s (%s bits, debug: %s)" % (self.minfo.get('version', False), self.minfo.get('bits', ''), self.minfo.get('debug', '')) )
isMaster = self.minfo.get("isMaster", None)
isreplicaset = isMaster.get("isreplicaset", False)
hosts = isMaster.get("hosts", [])
arbiters = isMaster.get("arbiters", [])
if self.setName:
cprint(" + setName:", str_state(self.setName==self.replSet.name, ok=self.setName, nok=self.setName))
cprint(" + Me:", self.me)
cprint(" + Master:", str_state(self.isMaster, ok="Yes", nok="No"))
cprint(" + Primary:", str_state(self.isPrimary, ok="Yes", nok="No"))
cprint(" + Secondary:", str_state(self.isSecondary, ok="Yes", nok="No"))
cprint(" + Arbiter:", str_state(self.isArbiter, ok="Yes", nok="No"))
cprint(" + Arbiters:", arbiters)
cprint(" + Members:", hosts)
"""
if not self.isPrimary and not self.isSecondary:
return
dbs = self.mclient.database_names()
cprint(" + DBs:", len(dbs))
for db in dbs:
cprint(" - '%s':" % db, "%s Collections" % len(self.mclient[db].collection_names()) )
"""
def __del__(self):
self.disconnect()
def check(self, agentConfig):
"""
Returns a dictionary that looks a lot like what's sent back by db.serverStatus()
"""
if 'mongodb_server' not in agentConfig or agentConfig['mongodb_server'] == '':
return False
try:
from pymongo import Connection
dbName = None
try:
from pymongo import uri_parser
# Configuration a URL, mongodb://user:pass@server/db
dbName = uri_parser.parse_uri(agentConfig['mongodb_server'])['database']
# parse_uri gives a default database of None
dbName = dbName or 'test'
except ImportError:
# uri_parser is pymongo 2.0+
dbName = mongo_uri_re.match(agentConfig['mongodb_server']).group(1)
if dbName is None:
self.logger.error("Mongo: cannot extract db name from config %s" % agentConfig['mongodb_server'])
return False
conn = Connection(agentConfig['mongodb_server'])
db = conn[dbName]
status = db.command('serverStatus') # Shorthand for {'serverStatus': 1}
status['stats'] = db.command('dbstats')
results = {}
# Handle replica data, if any
# See http://www.mongodb.org/display/DOCS/Replica+Set+Commands#ReplicaSetCommands-replSetGetStatus
try:
data = {}
replSet = conn['admin'].command('replSetGetStatus')
serverVersion = conn.server_info()['version']
if replSet:
primary = None
current = None
# find nodes: master and current node (ourself)
for member in replSet.get('members'):
if member.get('self'):
current = member
if int(member.get('state')) == 1:
primary = member
# If we have both we can compute a lag time
if current is not None and primary is not None:
lag = current['optimeDate'] - primary['optimeDate']
# Python 2.7 has this built in, python < 2.7 don't...
if hasattr(lag,'total_seconds'):
data['replicationLag'] = lag.total_seconds()
else:
data['replicationLag'] = (lag.microseconds + \
(lag.seconds + lag.days * 24 * 3600) * 10**6) / 10.0**6
if current is not None:
data['health'] = current['health']
data['state'] = replSet['myState']
event = self.checkLastState(data['state'], agentConfig, serverVersion)
if event is not None:
results['events'] = {'Mongo': [event]}
status['replSet'] = data
except:
self.logger.exception("Cannot determine replication set status")
# If these keys exist, remove them for now as they cannot be serialized
try:
status['backgroundFlushing'].pop('last_finished')
except KeyError:
pass
try:
status.pop('localTime')
except KeyError:
pass
# Flatten the metrics first
# Collect samples
# Send a dictionary back
for m in self.get_metric_names():
# each metric is of the form: x.y.z with z optional
# and can be found at status[x][y][z]
value = status
try:
for c in m.split("."):
value = value[c]
except KeyError:
continue
# value is now status[x][y][z]
assert type(value) in (types.IntType, types.LongType, types.FloatType)
self.save_sample(m, value)
# opposite op: x.y.z -> results[x][y][zPS], yes, ...PS for counters
try:
val = self.get_sample(m)
r = results
for c in m.split(".")[:-1]:
if c not in r:
r[c] = {}
r = r[c]
if self.is_counter(m):
suffix = m.split(".")[-1] + "PS"
else:
suffix = m.split(".")[-1]
r[suffix] = val
except UnknownValue:
pass
return results
except ImportError:
self.logger.exception('Unable to import pymongo library')
return False
except:
self.logger.exception('Unable to get MongoDB status')
return False
class MongoMonitor(object):
"""
MongoDB Monitor class. It sets daemon to store into MongoDB its
own stat information and provide web functionality.
"""
def __init__(self, dburi, dbname='mongodb', dbcoll='stats',
size=10*1024*1024, interval=5):
self.dburi = dburi
self.conn = Connection(dburi)
if dbname not in self.conn.database_names():
dbptr = self.conn[dbname]
dbptr.create_collection(dbcoll, {'capped':True, 'size':size})
self.coll = self.conn[dbname][dbcoll]
self.attr = []
for key, val in self.conn[dbname].command( { "serverStatus" : 1 } ).items():
if isinstance(val, dict):
for akey, aval in val.items():
if isinstance(aval, dict):
for kkk, vvv in aval.items():
if not isinstance(vvv, dict):
self.attr.append('%s.%s.%s' % (key, akey, kkk))
else:
self.attr.append('%s.%s' % (key, akey))
# start thread with db_updater
thread.start_new_thread(db_updater, (dburi, dbname, dbcoll, interval))
# setup js/css dirs
self.jsdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'js')
if os.environ.has_key('JSPATH'):
self.jsdir = os.environ['JSPATH']
if not os.path.isdir(self.jsdir):
raise Exception('JS path is not set')
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
'request.show_tracebacks': False,
'server.environment':'production',
'server.socket_host':'0.0.0.0',
})
self._cache = {}
def server_info(self):
"""Get server info"""
return self.conn.server_info()
def db_info(self):
"""
Get DB info
"""
ddict = {}
for database in self.conn.database_names():
collections = self.conn[database].collection_names()
coll = self.conn[database]
info_dict = {}
for cname in collections:
info_dict[cname] = coll.command({'collStats': cname})
ddict[database] = info_dict
return ddict
def get_stats(self, attr, datetime1, datetime2):
"""
Get MongoDB statistics for provided time interval
"""
spec = {'localTime' : {'$gte': datetime1, '$lte': datetime2}}
alist = attr.split('.')
if len(alist) == 2:
key, att = alist
alast = None
elif len(alist) == 3:
key, att, alast = alist
else:
raise Exception('Too many attributes')
for row in self.coll.find(spec):
if row.has_key(key) and row[key].has_key(att):
value = row[key][att]
if alast:
value = value[alast]
if isinstance(value, datetime.datetime):
value = time.mktime(datetime.datetime.timetuple(value))
if alast:
rec = {key : {att: {alast: value}}}
else:
rec = {key : {att: value}}
yield rec
def minmaxval(self, attr, datetime1, datetime2):
"""
Get MongoDB statistics for provided time interval
"""
spec = {'localTime' : {'$gte': datetime1, '$lte': datetime2}}
maxval = 0
minval = 0
alist = attr.split('.')
if len(alist) == 2:
key, att = alist
alast = None
elif len(alist) == 3:
key, att, alast = alist
else:
raise Exception('Too many attributes')
for row in self.coll.find(spec):
value = row[key][att]
if alast:
value = value[alast]
if isinstance(value, datetime.datetime):
value = time.mktime(datetime.datetime.timetuple(value))
if not minval:
minval = value
if value > maxval:
maxval = value
if value < minval:
minval = value
return minval, maxval
@expose
def index(self):
"""Main page"""
page = template_server_info(self.server_info())
page += template_db_info(self.db_info())
page += template_plot_form(self.attr)
return self.page('MongoDB', page)
@expose
def page(self, title, content):
"""Format HTML page"""
return template_top(title, self.dburi) + content + template_bottom()
@expose
def stat(self, **kwds):
"""Plot given set of parameters"""
time1 = kwds.get('t1')
time2 = kwds.get('t2')
if not time1 and not time2: # default is 1 hour
time1 = datetime.datetime.utcfromtimestamp(time.time()-3600)
time2 = datetime.datetime.utcfromtimestamp(time.time())
else:
time1 = parse_timestamp(time1)
time2 = parse_timestamp(time2)
attr = kwds.get('attr')
if attr not in self.attr:
raise HTTPError(500, 'Unknown attribute %s, supported: %s' \
% (attr, self.attr))
data = [r for r in self.get_stats(attr, time1, time2)]
minval, maxval = self.minmaxval(attr, time1, time2)
units, xmin, xmax = delta(time1, time2)
spec = {'data':str(data).replace("u'", "'"), 'attr':attr,
'title':attr, 'ymax':maxval, 'ymin':minval,
'units': units, 'xmin': xmin, 'xmax': xmax,
'time1': convert_timestamp(time1),
'time2': convert_timestamp(time2)}
content = template_plot(spec)
page = template_server_info(self.server_info())
page += template_db_info(self.db_info())
page += template_plot_form(self.attr, attr)
page += content
return self.page('MongoDB', page)
@exposejs
@tools.gzip()
def js(self, *args, **kwargs):
"""
Serve protovis JS file.
"""
args = ['/'.join(args)] # preserve YUI dir structure
scripts = self.check_scripts(args, self.jsmap, self.jsdir)
return self.serve_files(args, scripts, self.jsmap)
def check_scripts(self, scripts, resource, path):
"""
Check a script is known to the resource map
and that the script actually exists
"""
for script in scripts:
if script not in resource.keys():
spath = os.path.normpath(os.path.join(path, script))
if os.path.isfile(spath):
resource.update({script: spath})
return scripts
def serve_files(self, args, scripts, resource, datatype='', minimize=False):
"""
Return asked set of files for JS, CSS.
"""
idx = "-".join(scripts)
if idx not in self._cache.keys():
data = ''
if datatype == 'css':
data = '@CHARSET "UTF-8";'
for script in args:
path = os.path.join(sys.path[0], resource[script])
path = os.path.normpath(path)
ifile = open(path)
data = "\n".join ([data, ifile.read().\
replace('@CHARSET "UTF-8";', '')])
ifile.close()
if datatype == 'css':
set_headers("text/css")
if minimize:
self._cache[idx] = minify(data)
else:
self._cache[idx] = data
return self._cache[idx]
class Main:
pkt_sum = {}
pkt_sum_agg = {}
def __init__(self, config):
self.config = config
self.db_connect()
self.lock = threading.Lock()
self.local_ip = self.get_ip()
self.networks = [
{'re':re.compile('192.168.\d+.\d+'), 'zone':'internal'},
{'re':re.compile('10.\d+.\d+.\d+'), 'zone':'internal'},
{'re':re.compile('172.16.\d+.\d+'), 'zone':'internal'},
{'re':re.compile('169.254.\d+.\d+'), 'zone':'autoconf'},
]
# setup log
logging.basicConfig(filename=self.get_config('log'), level=logging.DEBUG, format='%(asctime)s %(message)s')
def get_config(self, name, section='general', default=''):
return self.config.get(section, name) if self.config.has_option(section, name) else default
def find_zone(self, ip):
for n in self.networks:
if re.match(n['re'], ip):
return n['zone']
return 'default'
# DB
def db_connect(self):
try:
self.connection = Connection(self.get_config('server'), 27017)
db = self.connection.network
self.flows = db.flows
except Exception as e:
logging.error("Connection error [%s]" % str(e))
def db_insert(self, doc):
try:
si = self.connection.server_info()
#print "Server Info [%s]" % str(si)
except:
self.db_connect()
try:
self.flows.insert(doc)
except Exception as e:
logging.error("Insert error [%s]" % str(e))
# Save aggragated data to DB
def dump(self):
if self.pkt_sum:
self.lock.acquire()
try:
msg = {'ts':int(time.time()), 'data':self.pkt_sum}
logging.debug(str(msg))
tosave_docs = []
for key, vals in self.pkt_sum.items():
keys = key.split("-")
doc = {'v': 0.2, 'ts':int(time.time()), 'src':keys[0], 'dst':keys[1], 'vals':vals, 'sum':self.pkt_sum_agg[key]}
#print doc
tosave_docs.append(doc)
self.pkt_sum = {}
self.pkt_sum_agg = {}
except Exception as e:
logging.error("Error in dump(): [%s]" % str(e))
finally:
self.lock.release()
# Insert in DB after lock release
try:
for doc in tosave_docs:
self.db_insert(doc)
except Exception as e:
logging.error("Error while saving data: [%s]" % str(e))
t = threading.Timer(60.0, self.dump)
t.daemon = True
t.start()
def get_ip(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('google.com', 0))
return s.getsockname()[0]
# Network listening
def go(self):
#self.dump()
try:
pc = pcap.pcap(name=self.get_config('iface'))
except Exception as e:
logging.error(str(e))
logging.error(str(e))
return False
filters = []
if self.get_config('mode')=="local":
filters.append("(src %s or dst %s)" % (self.local_ip, self.local_ip))
if self.get_config('mode')=="out":
filters.append("src %s" % (self.local_ip))
if self.get_config('filter')<>"":
filters.append(self.get_config('filter'))
filter_str = " and ".join(filters)
print filter_str
logging.info("Start listening with the following params:")
logging.info("iface [%s]" % self.get_config('iface'))
logging.info("filter [%s]" % filter_str)
pc.setfilter(filter_str)
for ts, pkt in pc:
p = dpkt.ethernet.Ethernet(pkt)
ip = p.data
#print `ip`
if p.type==dpkt.ethernet.ETH_TYPE_IP: # IP
src = socket.inet_ntoa(ip.src)
dst = socket.inet_ntoa(ip.dst)
size = int(ip.len)
proto_ip = str(ip.p)
sport = "0"
dport = "0"
if ip.p==dpkt.ip.IP_PROTO_TCP:
proto_ip = 'TCP'
sport = str(ip.data.sport)
dport = str(ip.data.dport)
elif ip.p==dpkt.ip.IP_PROTO_UDP:
proto_ip = 'UDP'
sport = str(ip.data.sport)
dport = str(ip.data.dport)
elif ip.p==dpkt.ip.IP_PROTO_IP6:
proto_ip = 'IPv6'
src_zone = self.find_zone(src)
dst_zone = self.find_zone(dst)
#print "%d: %s(%s) => %s(%s) [%d] [ip:%s] [dport:%d]" % (int(ts), src, src_zone, dst, dst_zone, size, proto_ip, dport)
print "Ethernet [%s]=>[%s] [%s] [%s]=>[%s:%s] [%d]" % (add_colons_to_mac(binascii.hexlify(p.src)), add_colons_to_mac(binascii.hexlify(p.dst)), proto_ip, src, dst, dport, size)
key = (src_zone if src_zone=="default" else src) + "-" + (dst_zone if dst_zone=="default" else dst)
self.lock.acquire()
try:
if key not in self.pkt_sum: self.pkt_sum[key] = {}
if proto_ip not in self.pkt_sum[key]: self.pkt_sum[key][proto_ip] = {'dport':{}, 'sport':{}}
if dport not in self.pkt_sum[key][proto_ip]['dport']: self.pkt_sum[key][proto_ip]['dport'][dport] = 0
if sport not in self.pkt_sum[key][proto_ip]['sport']: self.pkt_sum[key][proto_ip]['sport'][sport] = 0
self.pkt_sum[key][proto_ip]["dport"][dport] += size
self.pkt_sum[key][proto_ip]["sport"][sport] += size
#print self.pkt_sum
if key not in self.pkt_sum_agg: self.pkt_sum_agg[key] = {}
if proto_ip not in self.pkt_sum_agg[key]: self.pkt_sum_agg[key][proto_ip] = 0
self.pkt_sum_agg[key][proto_ip] += size
except Exception as e:
print str(e)
logging.error("Error in go(): [%s]" % str(e))
finally:
self.lock.release()
def check(self, agentConfig):
"""
Returns a dictionary that looks a lot like what's sent back by db.serverStatus()
"""
if 'mongodb_server' not in agentConfig or agentConfig[
'mongodb_server'] == '':
return False
try:
from pymongo import Connection
dbName = None
try:
from pymongo import uri_parser
# Configuration a URL, mongodb://user:pass@server/db
dbName = uri_parser.parse_uri(
agentConfig['mongodb_server'])['database']
# parse_uri gives a default database of None
dbName = dbName or 'test'
except ImportError:
# uri_parser is pymongo 2.0+
dbName = mongo_uri_re.match(
agentConfig['mongodb_server']).group(1)
if dbName is None:
self.logger.error(
"Mongo: cannot extract db name from config %s" %
agentConfig['mongodb_server'])
return False
conn = Connection(agentConfig['mongodb_server'])
db = conn[dbName]
status = db.command(
'serverStatus') # Shorthand for {'serverStatus': 1}
status['stats'] = db.command('dbstats')
results = {}
# Handle replica data, if any
# See http://www.mongodb.org/display/DOCS/Replica+Set+Commands#ReplicaSetCommands-replSetGetStatus
try:
data = {}
replSet = conn['admin'].command('replSetGetStatus')
serverVersion = conn.server_info()['version']
if replSet:
primary = None
current = None
# find nodes: master and current node (ourself)
for member in replSet.get('members'):
if member.get('self'):
current = member
if int(member.get('state')) == 1:
primary = member
# If we have both we can compute a lag time
if current is not None and primary is not None:
lag = current['optimeDate'] - primary['optimeDate']
# Python 2.7 has this built in, python < 2.7 don't...
if hasattr(lag, 'total_seconds'):
data['replicationLag'] = lag.total_seconds()
else:
data['replicationLag'] = (lag.microseconds + \
(lag.seconds + lag.days * 24 * 3600) * 10**6) / 10.0**6
if current is not None:
data['health'] = current['health']
data['state'] = replSet['myState']
event = self.checkLastState(data['state'], agentConfig,
serverVersion)
if event is not None:
results['events'] = {'Mongo': [event]}
status['replSet'] = data
except:
self.logger.exception(
"Cannot determine replication set status")
# If these keys exist, remove them for now as they cannot be serialized
try:
status['backgroundFlushing'].pop('last_finished')
except KeyError:
pass
try:
status.pop('localTime')
except KeyError:
pass
# Flatten the metrics first
# Collect samples
# Send a dictionary back
for m in self.get_metric_names():
# each metric is of the form: x.y.z with z optional
# and can be found at status[x][y][z]
value = status
try:
for c in m.split("."):
value = value[c]
except KeyError:
continue
# value is now status[x][y][z]
assert type(value) in (types.IntType, types.LongType,
types.FloatType)
self.save_sample(m, value)
# opposite op: x.y.z -> results[x][y][zPS], yes, ...PS for counters
try:
val = self.get_sample(m)
r = results
for c in m.split(".")[:-1]:
if c not in r:
r[c] = {}
r = r[c]
if self.is_counter(m):
suffix = m.split(".")[-1] + "PS"
else:
suffix = m.split(".")[-1]
r[suffix] = val
except UnknownValue:
pass
return results
except ImportError:
self.logger.exception('Unable to import pymongo library')
return False
except:
self.logger.exception('Unable to get MongoDB status')
return False
# getopt
try:
opts, args = getopt.getopt(sys.argv[1:], 'h:p:u:w:')
except getopt.GetoptError, err:
print str(err)
usage()
sys.exit(2)
for o, a in opts:
if o == "-h":
host = a
elif o == "-p":
port = int(a)
elif o == "-u":
username = a
elif o == "-w":
password = a
connection = Connection(host, port)
if not username is None:
connection['admin'].authenticate(username, password)
db = connection['admin']
print connection.server_info()
dblocal = connection['local']
#dblocal.system.namespaces.find({'name': 'local.oplog.$main'})
a = dblocal['optlog.$main'].find()
for op in a:
print a
