def launch(self):
logger.debug('getRabbitMQStatus: start')
if 'rabbitMQStatusUrl' not in self.config or \
'rabbitMQUser' not in self.config or \
'rabbitMQPass' not in self.config or \
self.config['rabbitMQStatusUrl'] == 'http://www.example.com:55672/json':
logger.debug('getRabbitMQStatus: config not set')
return False
logger.debug('getRabbitMQStatus: config set')
try:
logger.debug('getRabbitMQStatus: attempting authentication setup')
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
manager.add_password(None, self.config['rabbitMQStatusUrl'], self.config['rabbitMQUser'], self.config['rabbitMQPass'])
handler = urllib2.HTTPBasicAuthHandler(manager)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
logger.debug('getRabbitMQStatus: attempting urlopen')
req = urllib2.Request(self.config['rabbitMQStatusUrl'], None, {})
# Do the request, log any errors
request = urllib2.urlopen(req)
response = request.read()
except urllib2.HTTPError, e:
logger.error('Unable to get RabbitMQ status - HTTPError = %s', e)
return False
def check_parallel_run(self):
# TODO: other daemon run on nt
if os.name == 'nt':
logger.warning("The parallel daemon check is not available on nt")
self.open_pidfile(write=True)
return
# First open the pid file in open mode
self.open_pidfile()
try:
buf = self.fpid.readline().strip(' \r\n')
if not buf: # pid file was void, cool
return
pid = int(buf)
except Exception as err:
logger.info("Stale pidfile exists at %s (%s). Reusing it." % (err, self.lock_path))
return
try:
os.kill(pid, 0)
except Exception as err: # consider any exception as a stale pidfile.
# this includes :
# * PermissionError when a process with same pid exists but is executed by another user.
# * ProcessLookupError: [Errno 3] No such process.
logger.info("Stale pidfile exists (%s), Reusing it." % err)
return
logger.error("Valid previous daemon exists (pid=%s) Exiting." % pid)
raise SystemExit(2)
def launch(self):
#logger.debug('getIOStats: start')
iostats = {}
if sys.platform != 'linux2':
logger.debug('getIOStats: unsupported platform')
return False
#logger.debug('getIOStats: linux2')
headerRegexp = re.compile(r'([%\\/\-\_a-zA-Z0-9]+)[\s+]?')
itemRegexp = re.compile(r'^([a-zA-Z0-9\/]+)')
valueRegexp = re.compile(r'\d+\.\d+')
try:
_cmd = 'iostat -d 1 2 -x -k'
stats = self.execute_shell(_cmd)
if not stats:
logger.error('getIOStats: exception in launching command')
return False
recentStats = stats.split('Device:')[2].split('\n')
header = recentStats[0]
headerNames = re.findall(headerRegexp, header)
device = None
for statsIndex in range(1, len(recentStats)):
row = recentStats[statsIndex]
if not row:
# Ignore blank lines.
continue
deviceMatch = re.match(itemRegexp, row)
if deviceMatch is not None:
# Sometimes device names span two lines.
device = deviceMatch.groups()[0]
values = re.findall(valueRegexp, row.replace(',', '.'))
if not values:
# Sometimes values are on the next line so we encounter
# instances of [].
continue
iostats[device] = {}
for headerIndex in range(0, len(headerNames)):
headerName = headerNames[headerIndex]
iostats[device][headerName] = float(values[headerIndex])
except Exception:
logger.error('getIOStats: exception = %s', traceback.format_exc())
return False
#logger.debug('getIOStats: completed, returning')
return iostats
def load_package(self, package, file_path):
logger.debug('Loading package data from file %s' % file_path)
pname = package.get('name', None)
if pname is None:
logger.error('Package data is missing name entry (%s)' % file_path)
return None
self.packs[pname] = package
return pname
def do_leave(name=""):
# Lookup at the localhost name first
if not name:
try:
(code, r) = get_kunai_local("/agent/name")
except request_errors, exp:
logger.error(exp)
return
name = r
def do_exec(tag="*", cmd="uname -a"):
if cmd == "":
logger.error("Missing command")
return
try:
(code, r) = get_kunai_local("/exec/%s?cmd=%s" % (tag, cmd))
except request_errors, exp:
logger.error(exp)
return
def do_join(seed=""):
if seed == "":
logger.error("Missing target argument. For example 192.168.0.1:6768")
return
try:
(code, r) = get_kunai_local("/agent/join/%s" % seed)
except request_errors, exp:
logger.error(exp)
return
def do_exec(tag='*', cmd='uname -a'):
if cmd == '':
logger.error('Missing command')
return
try:
(code, r) = get_kunai_local('/exec/%s?cmd=%s' % (tag, cmd))
except request_errors, exp:
logger.error(exp)
return
def do_join(seed=''):
if seed == '':
logger.error('Missing target argument. For example 192.168.0.1:6768')
return
try:
(code, r) = get_kunai_local('/agent/join/%s' % seed)
except request_errors, exp:
logger.error(exp)
return
def do_exec(tag='*', cmd='uname -a'):
if cmd == '':
logger.error('Missing command')
return
try:
r = rq.get('http://%s:6768/exec/%s?cmd=%s' % (HOSTNAME, tag, cmd))
except rq.exceptions.ConnectionError, exp:
logger.error(exp)
return
def do_state(name=""):
uri = "/agent/state/%s" % name
if not name:
uri = "/agent/state"
try:
(code, r) = get_kunai_local(uri)
except request_errors, exp:
logger.error(exp)
return
def do_leave(name=''):
# Lookup at the localhost name first
if not name:
try:
r = rq.get('http://%s:6768/agent/name' % HOSTNAME)
except rq.exceptions.ConnectionError, exp:
logger.error(exp)
return
name = r.text
def do_join(seed=''):
if seed == '':
logger.error('Missing target argument. For example 192.168.0.1:6768')
return
try:
r = rq.get('http://%s:6768/agent/join/%s' % (HOSTNAME, seed))
except rq.exceptions.ConnectionError, exp:
logger.error(exp)
return
def do_state(name=''):
uri = 'http://%s:6768/agent/state/%s' % (HOSTNAME, name)
if not name:
uri = 'http://%s:6768/agent/state' % HOSTNAME
try:
r = rq.get(uri)
except rq.exceptions.ConnectionError, exp:
logger.error(exp)
return
def launch(self):
logger.debug('getMySQLStatus: start')
# Try import MySQLdb - http://sourceforge.net/projects/mysql-python/files/
try:
import MySQLdb
except ImportError, e:
logger.error('Unable to import MySQLdb')
return False
def send_all(self, o):
try:
msg = json.dumps(o)
except ValueError:
return
for client in self.server.connections.itervalues():
try:
client.sendMessage(msg)
except Exception as exp:
logger.error('Cannot send websocket message: %s' % exp, part='websocket')
def load_directory(self, directory, pack_name=''):
logger.debug('Loading collector directory at %s for pack %s' % (directory, pack_name))
pth = directory + '/collector_*.py'
collector_files = glob.glob(pth)
for f in collector_files:
fname = os.path.splitext(os.path.basename(f))[0]
logger.debug('Loading collector from file %s' % f)
try:
m = imp.load_source('collector_%s_%s' % (pack_name, fname), f)
logger.debug('Collector module loaded: %s' % m)
except Exception, exp:
logger.error('Cannot load collector %s: %s' % (fname, exp))
def launch(self):
logger.debug('getMongoDBStatus: start')
if 'MongoDBServer' not in self.config or self.config['MongoDBServer'] == '':
logger.debug('getMongoDBStatus: config not set')
return False
logger.debug('getMongoDBStatus: config set')
try:
import pymongo
from pymongo import Connection
except ImportError:
logger.error('Unable to import pymongo library')
return False
# The dictionary to be returned.
mongodb = {}
try:
import urlparse
parsed = urlparse.urlparse(self.config['MongoDBServer'])
mongoURI = ''
# Can't use attributes on Python 2.4
if parsed[0] != 'mongodb':
mongoURI = 'mongodb://'
if parsed[2]:
if parsed[0]:
mongoURI = mongoURI + parsed[0] + ':' + parsed[2]
else:
mongoURI = mongoURI + parsed[2]
else:
mongoURI = self.config['MongoDBServer']
logger.debug('-- mongoURI: %s', mongoURI)
conn = Connection(mongoURI, slave_okay=True)
logger.debug('Connected to MongoDB')
except Exception, ex:
logger.error('Unable to connect to MongoDB server %s - Exception = %s', mongoURI, traceback.format_exc())
return False
def get_collectors(self):
collector_dir = os.path.dirname(__file__)
p = collector_dir + '/collectors/*py'
logger.debug('Loading collectors from ', p)
collector_files = glob.glob(p)
for f in collector_files:
fname = os.path.splitext(os.path.basename(f))[0]
try:
imp.load_source('collector%s' % fname, f)
except Exception, exp:
logger.error('Cannot load collector %s: %s' % (fname, exp))
continue
def w():
tid = 0
if libc:
tid = libc.syscall(186) # get the threadid when you are in it :)
logger.info('THREAD launch (%s) with thread id (%d)' % (name, tid))
try:
f(*args)
except Exception, exp:
output = cStringIO.StringIO()
traceback.print_exc(file=output)
logger.error("Thread %s is exiting on error. Back trace of this error: %s" % (name, output.getvalue()))
output.close()
def launch(self):
#logger.debug('get_open_ports: start')
open_ports = {'tcp': [], 'udp': []}
if sys.platform != 'linux2':
logger.debug('get_open_ports: unsupported platform')
return False
#logger.debug('get_open_ports: linux2')
try:
_cmd = 'netstat -tuln'
netstat = self.execute_shell(_cmd)
if not netstat:
logger.error('get_open_ports: exception in launching command')
return False
for line in netstat.splitlines():
line = line.strip()
# Will be something like
#tcp 0 0 0.0.0.0:27017 0.0.0.0:* LISTEN
if not line.startswith('tcp') and not line.startswith('udp'):
# Not a good line, skip it
continue
#print "LOOKING AT LINE"
elts = [ e for e in line.split(' ') if e]
#print "ELEMENTS", elts
if len(elts) != 6:
#print "BAD LINE", elts
continue
open_port = {}
open_port['proto'] = elts[0]
open_port['source'] = elts[3]
open_port['dest'] = elts[4]
open_port['state'] = elts[5]
if open_port['proto'].startswith('tcp'):
open_ports['tcp'].append(open_port)
elif open_port['proto'].startswith('udp'):
open_ports['udp'].append(open_port)
else:
print "Unknown protocol??"
except Exception:
logger.error('get_open_ports: exception = %s', traceback.format_exc())
return False
#logger.debug('get_open_ports: completed, returning')
return open_ports
def launch(self):
logger.debug('getCPUStats: start')
cpuStats = {}
if sys.platform == 'linux2':
logger.debug('getCPUStats: linux2')
headerRegexp = re.compile(r'.*?([%][a-zA-Z0-9]+)[\s+]?')
itemRegexp = re.compile(r'.*?\s+(\d+)[\s+]?')
itemRegexpAll = re.compile(r'.*?\s+(all)[\s+]?')
valueRegexp = re.compile(r'\d+\.\d+')
try:
cmd = 'mpstat -P ALL 1 1'
stats = self.execute_shell(cmd)
if not stats:
return None
stats = stats.split('\n')
header = stats[2]
headerNames = re.findall(headerRegexp, header)
device = None
for statsIndex in range(3, len(stats)): # no skip "all"
row = stats[statsIndex]
if not row: # skip the averages
break
deviceMatchAll = re.match(itemRegexpAll, row)
deviceMatch = re.match(itemRegexp, row)
if deviceMatchAll is not None:
device = 'cpuall'
elif deviceMatch is not None:
device = 'cpu%s' % deviceMatch.groups()[0]
values = re.findall(valueRegexp, row.replace(',', '.'))
cpuStats[device] = {}
for headerIndex in range(0, len(headerNames)):
headerName = headerNames[headerIndex]
cpuStats[device][headerName] = float(values[headerIndex])
except Exception:
logger.error('getCPUStats: exception = %s', traceback.format_exc())
return False
else:
logger.debug('getCPUStats: unsupported platform')
return False
logger.debug('getCPUStats: completed, returning')
return cpuStats
def encrypt(self, data):
if not self.encryption_key:
return data
logger.debug('ENCRYPT with '+self.encryption_key)
# Be sure the data is x16 lenght
if len(data) % 16 != 0:
data += ' ' * (-len(data) % 16)
try:
cyph = AES.new(self.encryption_key, AES.MODE_ECB)
ndata = cyph.encrypt(data)
return ndata
except Exception, exp:
logger.error('Encryption fail:', exp, part='gossip')
return ''
def launch(self):
now = int(time.time())
diff = now - self.last_launch
self.last_launch = now
logger.debug('getKernelStats: start')
if sys.platform == 'linux2':
logger.debug('getKernelStats: linux2')
try:
logger.debug('getKernelStats: attempting open')
lines = []
with open('/proc/stat', 'r') as stats:
lines.extend(stats.readlines())
with open('/proc/vmstat', 'r') as vmstat:
lines.extend(vmstat.readlines())
except IOError, e:
logger.error('getKernelStat: exception = %s', e)
return False
logger.debug('getKernelStat: open success, parsing')
data = {}
for line in lines:
elts = line.split(' ', 1)
# only look at keys
if len(elts) != 2:
continue
try:
data[elts[0]] = long(elts[1])
except ValueError: # not an int? skip this value
continue
# Now loop through each interface
by_sec_keys = ['ctxt', 'processes', 'pgfault', 'pgmajfault']
to_add = {}
for (k,v) in data.iteritems():
if k in by_sec_keys:
if k in self.store:
to_add['%s/s' % k] = (v - self.store[k]) / diff
else:
to_add['%s/s' % k] = 0
self.store[k] = data[k]
for k in by_sec_keys:
del data[k]
data.update(to_add)
logger.debug('getKernelStats: completed, returning')
return data
def generate(self, clust):
# HACK, atomise this global thing!!
global gclust
gclust = clust
# If not jinja2, bailing out
if jinja2 is None:
return
try:
f = open(self.g['template'], 'r')
self.buf = f.read()
f.close()
except IOError, exp:
logger.error('Cannot open template file %s : %s' % (self.g['template'], exp))
self.buf = None
self.template = None
def load_collector(self, cls):
colname = cls.__name__.lower()
# If already loaded, skip it
if colname in self.collectors:
return
logger.debug('Loading collector %s from class %s' % (colname, cls))
try:
# also give it our put result callback
inst = cls(self.cfg_data, put_result=self.put_result)
except Exception, exp:
logger.error('Cannot load the %s collector: %s' % (cls, traceback.format_exc()))
return
def daemonize(self):
logger.debug("Redirecting stdout and stderr as necessary..")
if self.debug_path:
fdtemp = os.open(self.debug_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC)
else:
fdtemp = os.open(REDIRECT_TO, os.O_RDWR)
os.dup2(fdtemp, 1) # standard output (1)
os.dup2(fdtemp, 2) # standard error (2)
# Now the fork/setsid/fork..
try:
pid = os.fork()
except OSError, e:
s = "%s [%d]" % (e.strerror, e.errno)
logger.error(s)
raise Exception, s
def write_if_need(self):
# Maybe there is no output, if so bail out now :)
if self.output is None:
return False
self.cur_value = ''
# first try to load the current file if exist and compare to the generated file
if os.path.exists(self.g['path']):
try:
f = open(self.g['path'], 'r')
self.cur_value = f.read()
f.close()
except IOError, exp:
logger.error('Cannot open path file %s : %s' % (self.g['path'], exp))
self.output = None
self.template = ''
self.buf = ''
return False
def do_keygen():
k = uuid.uuid1().hex[:16]
cprint("UDP Encryption key: (aka encryption_key)", end="")
cprint(base64.b64encode(k), color="green")
print ""
try:
import rsa
except ImportError:
logger.error("Missing python-rsa module for RSA keys generation, please install it")
return
pubkey, privkey = rsa.newkeys(2048)
print "Private RSA key (2048). (aka master_key_priv for for file mfkey.priv)"
s_privkey = privkey.save_pkcs1()
cprint(s_privkey, color="green")
print ""
print "Public RSA key (2048). (aka master_key_pub for file mfkey.pub)"
s_pubkey = pubkey.save_pkcs1()
cprint(s_pubkey, color="green")
print ""
def do_keygen():
k = uuid.uuid1().hex[:16]
cprint('UDP Encryption key: (aka encryption_key)', end='')
cprint(base64.b64encode(k), color='green')
print ''
try:
from Crypto.PublicKey import RSA
except ImportError:
logger.error('Missing python-crypto module for RSA keys generation, please install it')
return
key = RSA.generate(2048)
privkey = key.exportKey()
pub_key = key.publickey()
pubkey = pub_key.exportKey()
print "Private RSA key (2048). (aka master_key_priv for for file mfkey.priv)"
cprint(privkey, color='green')
print ''
print "Public RSA key (2048). (aka master_key_pub for file mfkey.pub)"
cprint(pubkey, color='green')
print ''