def __init__(self, config, handlers):
super(CephCollector, self).__init__(config, handlers)
self.config["short_names"] = str_to_bool(self.config["short_names"])
self.config["service_stats_global"] = str_to_bool(self.config["service_stats_global"])
self.config["perf_counters_enabled"] = str_to_bool(self.config["perf_counters_enabled"])
self.config["osd_stats_enabled"] = str_to_bool(self.config["osd_stats_enabled"])
self.config["long_running_detail"] = str_to_bool(self.config["long_running_detail"])
def collect(self):
"""
Collect and publish netfilter counters
"""
cmd = [self.config['bin'], "list"]
if str_to_bool(self.config['reset']):
cmd.append("reset")
if str_to_bool(self.config['use_sudo']):
cmd.insert(0, self.config['sudo_cmd'])
# We avoid use of the XML format to mtaintain compatbility with older
# versions of nfacct and also to avoid the bug where pkts and bytes were
# flipped
# Each line is of the format:
# { pkts = 00000000000001121700, bytes = 00000000000587037355 } = ipv4;
matcher = re.compile("{ pkts = (.*), bytes = (.*) } = (.*);")
lines = Popen(cmd, stdout=PIPE).communicate()[0].strip().splitlines()
for line in lines:
matches = re.match(matcher, line)
if matches:
num_packets = int(matches.group(1))
num_bytes = int(matches.group(2))
name = matches.group(3)
self.publish(name + ".pkts", num_packets)
self.publish(name + ".bytes", num_bytes)
def __init__(self, *args, **kwargs):
super(MySQLCollector, self).__init__(*args, **kwargs)
for key in self.innodb_status_keys:
self.innodb_status_keys[key] = re.compile(
self.innodb_status_keys[key])
if self.config['hosts'].__class__.__name__ != 'list':
self.config['hosts'] = [self.config['hosts']]
# Move legacy config format to new format
if 'host' in self.config:
hoststr = "%s:%s@%s:%s/%s" % (
self.config['user'],
self.config['passwd'],
self.config['host'],
self.config['port'],
self.config['db'],
)
self.config['hosts'].append(hoststr)
# Normalize some config vars
self.config['master'] = str_to_bool(self.config['master'])
self.config['slave'] = str_to_bool(self.config['slave'])
self.config['innodb'] = str_to_bool(self.config['innodb'])
self.db = None
def collect(self):
data = self.get_data()
if not data:
return
if str_to_bool(self.config['include_clients']):
metric_name = 'postfix.incoming'
if not str_to_bool(self.config['relay_mode']):
for client, value in data.get('clients', {}).iteritems():
self.dimensions = {
'client': str(client),
}
self.publish_cumulative_counter(metric_name, value)
else:
for component, clients in data.get('relay_clients', {}).iteritems():
for client, value in clients.iteritems():
self.dimensions = {
'client': str(client),
'queue':str(component),
}
self.publish_cumulative_counter(metric_name, value)
for action in (u'in', u'recv', u'send'):
if action not in data:
continue
metric_name = '.'.join(['postfix', str(action)])
for sect, components in data[action].iteritems():
if not str_to_bool(self.config['relay_mode']):
if sect == 'relay_status':
continue
for status, value in components.iteritems():
self.dimensions = {
'status': str(status),
}
self.publish_cumulative_counter(metric_name, value)
else:
if sect != 'relay_status':
continue
for component, stats in components.iteritems():
for status, value in stats.iteritems():
self.dimensions = {
'status': str(status),
'queue':str(component),
}
self.publish_cumulative_counter(metric_name, value)
if u'local' in data:
metric_name = 'postfix.local'
for key, value in data[u'local'].iteritems():
self.dimensions = {'address': str(key)}
self.publish_cumulative_counter(metric_name, value)
def collect_bean(self, prefix, obj):
for k, v in obj.iteritems():
if type(v) in [int, float, long]:
self.parse_dimension_bean(prefix, k, v)
elif isinstance(v, dict) and str_to_bool(self.config["nested"]):
self.collect_bean("%s.%s" % (prefix, k), v)
elif isinstance(v, list) and str_to_bool(self.config["nested"]):
self.interpret_bean_with_list("%s.%s" % (prefix, k), v)
def collect(self):
"""
Do pre-flight checks, get list of db names, collect metrics, publish
"""
if psycopg2 is None:
self.log.error('Unable to import module psycopg2')
return {}
# Get list of databases
dbs = self._get_db_names()
if len(dbs) == 0:
self.log.error("I have 0 databases!")
return {}
if self.config['metrics']:
metrics = self.config['metrics']
elif str_to_bool(self.config['extended']):
metrics = registry['extended']
if str_to_bool(self.config['has_admin']) \
and 'WalSegmentStats' not in metrics:
metrics.append('WalSegmentStats')
else:
metrics = registry['basic']
# Iterate every QueryStats class
for metric_name in set(metrics):
if metric_name not in metrics_registry:
self.log.error(
'metric_name %s not found in metric registry' % metric_name)
continue
for dbase in dbs:
conn = self._connect(database=dbase)
try:
klass = metrics_registry[metric_name]
stat = klass(dbase, conn,
underscore=self.config['underscore'])
stat.fetch(self.config['pg_version'])
for metric, value in stat:
if value is not None:
self.publish(metric, value)
# Setting multi_db to True will run this query on all known
# databases. This is bad for queries that hit views like
# pg_database, which are shared across databases.
#
# If multi_db is False, bail early after the first query
# iteration. Otherwise, continue to remaining databases.
if stat.multi_db is False:
break
finally:
conn.close()
def _collect_from(self, filename):
"""Loops through each line in the file, parses the logged data into
metrics, and publishes the metrics."""
if not os.access(filename, os.R_OK):
self.log.error('HadoopMetrics2Collector unable to read "%s"', filename)
return
self.config['hostname_method'] = 'uname_short'
if str_to_bool(self.config['truncate']):
# It is too dangerous to truncate a file that may be in the process
# of being appended to; especially since most file systems do not
# provide functionality for removing data from the start of a file.
# As such we simply rename the file and delete the entire file when
# we are finished.
original_filename = filename
filename = os.path.join(os.path.dirname(original_filename),
self.__generate_unique_filename(RESERVED_NAME))
os.rename(original_filename, filename)
_file = open(filename, 'r')
for line in _file:
match = self.re_log.match(line)
if not match:
continue
raw_data = match.groupdict()
metrics = {}
extra_data = {}
for metric in raw_data['metrics'].split(','):
metric = metric.strip()
if '=' in metric:
key, value = metric.split('=', 1)
try:
metrics[key] = round(float(value))
except ValueError:
extra_data[key] = value
host = extra_data.get('Hostname', None) or self.get_hostname()
partial_path = 'hadoop.{}.'.format(raw_data['name'])
for key, val in metrics.items():
full_path = partial_path + key
self._publish(key, val, full_path, host, raw_data['timestamp'])
_file.close()
if str_to_bool(self.config['truncate']):
os.remove(filename)
def _collect_virtual(self):
command = [self.config['bin'], "-LDInfo", "-Lall", "-aALL", "-NoLog"]
if str_to_bool(self.config['use_sudo']):
command.insert(0, self.config['sudo_cmd'])
output = subprocess.Popen(
command,
stdout=subprocess.PIPE
).communicate()[0].strip().splitlines()
metrics = {}
adapter = 0
vd = 0
default_cache_policy = ""
for line in output:
line = line.strip()
if line == "":
continue
if line.startswith("Adapter "):
adapter = int(line[8:line.find(' ', 8)])
continue
if line.startswith("Virtual Drive: "):
vd = int(line[15:line.find(' ', 15)])
continue
if line.startswith("Exit Code: "):
continue
linedata = line.split(":")
if len(linedata) == 2:
key = linedata[0].strip()
value = linedata[1].strip()
if key == "RAID Level":
metrics["vd.adapter%d.virt%d.raid_level" % (adapter, vd)] = int(value[8:9])
if key == "State":
metrics["vd.adapter%d.virt%d.state_optimal" % (adapter, vd)] = int(value == "Optimal")
if key == "Number Of Drives":
metrics["vd.adapter%d.virt%d.drives" % (adapter, vd)] = int(value)
if key == "Default Cache Policy":
default_cache_policy = value
if key == "Current Cache Policy":
metrics["vd.adapter%d.virt%d.cache_policy_default" % (adapter, vd)] = int(value == default_cache_policy)
if key == "Bad Blocks Exist":
metrics["vd.adapter%d.virt%d.bad_blocks" % (adapter, vd)] = int(str_to_bool(value))
for name, metric in metrics.items():
self.publish(name, metric)
def collect(self):
command = [self.config['bin'], '-l']
if str_to_bool(self.config['use_sudo']):
command.insert(0, self.config['sudo_cmd'])
try:
p = subprocess.Popen(command, stdout=subprocess.PIPE)
res = p.communicate()[0]
except Exception as e:
self.log.error('Unable to exec cmd: %s, because %s'
% (' '.join(command), str(e)))
return
if res == '':
self.log.error('Empty result from exec cmd: %s'
% (' '.join(command)))
return
states = {}
for line in res.split("\n"):
# ID: 000, Name: local-ossec-001.localdomain (server), IP:\
# 127.0.0.1, Active/Local
if not line.startswith(' ID: '):
continue
fragments = line.split(',')
state = fragments[-1].lstrip()
if state not in states:
states[state] = 1
else:
states[state] += 1
for state, count in states.items():
name = 'agents.' + re.sub('[^a-z]', '_', state.lower())
self.publish(name, count)
def collect(self):
if not os.access(self.config['bin'], os.X_OK):
self.log.error("%s is not executable", self.config['bin'])
return False
command = [self.config['bin'], 'list']
if str_to_bool(self.config['use_sudo']):
command.insert(0, self.config['sudo_cmd'])
data = subprocess.Popen(command,
stdout=subprocess.PIPE).communicate()[0]
for metric in data.split(','):
if not metric.strip():
continue
metric, value = metric.split('=')
try:
value = float(value)
except:
pass
if metric not in self._GAUGE_KEYS:
value = self.derivative(metric, value)
if value < 0:
continue
self.publish(metric, value)
def __init__(self, *args, **kwargs):
super(MantridCollector, self).__init__(*args, **kwargs)
self.statcommand = [self.config['bin'], 'stats']
if str_to_bool(self.config['use_sudo']):
self.statcommand.insert(0, self.config['sudo_cmd'])
def collect(self):
if psycopg2 is None:
self.log.error('Unable to import module psycopg2')
return {}
# Create database-specific connections
self.connections = {}
for db in self._get_db_names():
self.connections[db] = self._connect(database=db)
if self.config['metrics']:
metrics = self.config['metrics']
elif str_to_bool(self.config['extended']):
metrics = registry['extended']
else:
metrics = registry['basic']
# Iterate every QueryStats class
for metric_name in set(metrics):
if metric_name not in metrics_registry:
continue
klass = metrics_registry[metric_name]
stat = klass(self.connections, underscore=self.config['underscore'])
stat.fetch()
for metric, value in stat:
if value is not None:
self.publish(metric, value)
# Cleanup
[conn.close() for conn in self.connections.itervalues()]
def collect(self):
if not os.access(self.config['bin'], os.X_OK):
self.log.error("%s is not executable", self.config['bin'])
return False
command = [self.config['bin'], self.config['ups_name']]
if str_to_bool(self.config['use_sudo']):
command.insert(0, self.config['sudo_cmd'])
p = subprocess.Popen(command,
stdout=subprocess.PIPE).communicate()[0]
for ln in p.strip().splitlines():
datapoint = ln.split(": ")
try:
val = float(datapoint[1])
except:
continue
if len(datapoint[0].split(".")) == 2:
# If the metric name is the same as the subfolder
# double it so it's visible.
name = ".".join([datapoint[0], datapoint[0].split(".")[1]])
else:
name = datapoint[0]
self.publish(name, val)
def get_passenger_memory_stats(self):
"""
Execute passenger-memory-stats, parse its output, return dictionary with
stats.
"""
command = [self.config["passenger_memory_stats_bin"]]
if str_to_bool(self.config["use_sudo"]):
command.insert(0, self.config["sudo_cmd"])
try:
proc1 = subprocess.Popen(command, stdout=subprocess.PIPE)
(std_out, std_err) = proc1.communicate()
except OSError:
return {}
if std_out is None:
return {}
dict_stats = {
"apache_procs": [],
"nginx_procs": [],
"passenger_procs": [],
"apache_mem_total": 0.0,
"nginx_mem_total": 0.0,
"passenger_mem_total": 0.0,
}
#
re_colour = re.compile("\x1B\[([0-9]{1,3}((;[0-9]{1,3})*)?)?[m|K]")
re_digit = re.compile("^\d")
#
apache_flag = 0
nginx_flag = 0
passenger_flag = 0
for raw_line in std_out.splitlines():
line = re_colour.sub("", raw_line)
if "Apache processes" in line:
apache_flag = 1
elif "Nginx processes" in line:
nginx_flag = 1
elif "Passenger processes" in line:
passenger_flag = 1
elif re_digit.match(line):
# If line starts with digit, then store PID and memory consumed
line_splitted = line.split()
if apache_flag == 1:
dict_stats["apache_procs"].append(line_splitted[0])
dict_stats["apache_mem_total"] += float(line_splitted[4])
elif nginx_flag == 1:
dict_stats["nginx_procs"].append(line_splitted[0])
dict_stats["nginx_mem_total"] += float(line_splitted[4])
elif passenger_flag == 1:
dict_stats["passenger_procs"].append(line_splitted[0])
dict_stats["passenger_mem_total"] += float(line_splitted[3])
elif "Processes:" in line:
passenger_flag = 0
apache_flag = 0
nginx_flag = 0
return dict_stats
def collect(self):
for key in self.config.keys():
if key[:7] == "target_":
host = self.config[key]
metric_name = key[7:] + '-' + host.replace('.', '_')
if not os.access(self.config['bin'], os.X_OK):
self.log.error("Path %s does not exist or is not executable"
% self.config['bin'])
return
command = [self.config['bin'], '-nq', '-c 1', '-W', '5', host]
if str_to_bool(self.config['use_sudo']):
command.insert(0, self.config['sudo_cmd'])
ping = subprocess.Popen(
command, stdout=subprocess.PIPE).communicate()[0].strip(
).split("\n")[-1]
# Linux
if ping.startswith('rtt'):
ping = ping.split()[3].split('/')[0]
metric_value = float(ping)
# OS X
elif ping.startswith('round-trip '):
ping = ping.split()[3].split('/')[0]
metric_value = float(ping)
# Unknown
else:
metric_value = 10000
self.publish(metric_name, metric_value)
def get_temp(self, device):
command = [self.config['bin'], '-n', device]
if str_to_bool(self.config['use_sudo']):
command.insert(0, self.config['sudo_cmd'])
return subprocess.Popen(command, stdout=subprocess.PIPE)
def collect(self):
if not os.access(self.config['bin'], os.X_OK):
self.log.error("%s is not executable", self.config['bin'])
return False
if (str_to_bool(self.config['use_sudo'])
and not os.access(self.config['sudo_cmd'], os.X_OK)):
self.log.error("%s is not executable", self.config['sudo_cmd'])
return False
client = subprocess.Popen(self.statcommand,
stdout=subprocess.PIPE).communicate()
columns = {
'open': 1,
'completed': 2,
'inbytes': 3,
'outbytes': 4,
}
for i, line in enumerate(client[0][:-1].split("\n")):
if i < 1:
continue
row = line.split()
host = string.replace(row[0], ".", "_")
external = host_under
for metric, column in columns.iteritems():
metric_name = ".".join([external, metric])
metric_value = row[column]
self.publish(metric_name, metric_value)
def collect(self):
load01, load05, load15 = os.getloadavg()
cpu_count = multiprocessing.cpu_count()
if not str_to_bool(self.config["simple"]):
self.publish_gauge("01", load01, 2)
self.publish_gauge("05", load05, 2)
self.publish_gauge("15", load15, 2)
self.publish_gauge("01_normalized", load01 / cpu_count, 2)
self.publish_gauge("05_normalized", load05 / cpu_count, 2)
self.publish_gauge("15_normalized", load15 / cpu_count, 2)
else:
self.publish_gauge("load", load01, 2)
self.publish_gauge("load_normalized", load01 / cpu_count, 2)
# Legacy: add process/thread counters provided by
# /proc/loadavg (if available).
if os.access(self.PROC_LOADAVG, os.R_OK):
file = open(self.PROC_LOADAVG)
for line in file:
match = self.PROC_LOADAVG_RE.match(line)
if match:
self.publish_gauge("processes_running", int(match.group(4)))
self.publish_gauge("processes_total", int(match.group(5)))
file.close()
def filter_metric(metric):
if not str_to_bool(self.config['simple']):
return True
else:
if metric.startswith('memory.stats.'):
if not metric.startswith('total', 13):
return False
return True
def collect(self):
load01, load05, load15 = os.getloadavg()
if not str_to_bool(self.config['simple']):
self.publish_gauge('01', load01, 2)
self.publish_gauge('05', load05, 2)
self.publish_gauge('15', load15, 2)
else:
self.publish_gauge('load', load01, 2)
def run_command(self, command):
try:
if str_to_bool(self.config["use_sudo"]):
command.insert(0, self.config["sudo_cmd"])
return subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0]
except OSError:
self.log.exception("Unable to run %s", command)
return ""
def report_cpu_metric(self, statname, value, instance):
# Value in cummulative nanoseconds
if str_to_bool(self.config['cpu_absolute']):
metric = value
else:
# Nanoseconds (10^9), however, we want to express in 100%
metric = self.derivative(statname, float(value) / 10000000.0,
max_value=diamond.collector.MAX_COUNTER,
instance=instance)
self.publish(statname, metric, instance=instance)
def collect(self):
"""
Collect and publish S.M.A.R.T. attributes
"""
devices = re.compile(self.config['devices'])
for device in os.listdir('/dev'):
if devices.match(device):
command = [self.config['bin'], "-A", os.path.join('/dev',
device)]
if str_to_bool(self.config['use_sudo']):
command.insert(0, self.config['sudo_cmd'])
attributes = subprocess.Popen(
command,
stdout=subprocess.PIPE
).communicate()[0].strip().splitlines()
metrics = {}
start_line = self.find_attr_start_line(attributes)
for attr in attributes[start_line:]:
attribute = attr.split()
if attribute[1] != "Unknown_Attribute":
metric = "%s.%s" % (device, attribute[1])
else:
metric = "%s.%s" % (device, attribute[0])
# 234 Thermal_Throttle (...) 0/0
if '/' in attribute[9]:
expanded = attribute[9].split('/')
for i, subattribute in enumerate(expanded):
submetric = '%s_%d' % (metric, i)
if submetric not in metrics:
metrics[submetric] = subattribute
elif metrics[submetric] == 0 and subattribute > 0:
metrics[submetric] = subattribute
else:
# New metric? Store it
if metric not in metrics:
metrics[metric] = attribute[9]
# Duplicate metric? Only store if it has a larger value
# This happens semi-often with the Temperature_Celsius
# attribute You will have a PASS/FAIL after the real
# temp, so only overwrite if The earlier one was a
# PASS/FAIL (0/1)
elif metrics[metric] == 0 and attribute[9] > 0:
metrics[metric] = attribute[9]
else:
continue
for metric in metrics.keys():
self.publish(metric, metrics[metric])
def collect(self):
for stat, val in self.get_scribe_stats():
metric_name = '.'.join(['scribe', stat])
self.log.debug(
"Publishing: {0} {1}".format(stat, val)
)
if str_to_bool(self.config['scribe_leaf']):
self.dimensions = { 'node_type': 'leaf' }
else:
self.dimensions = { 'node_type': 'aggregator' }
self.publish(metric_name, val)
def collect(self):
if citrusleaf is None:
self.log.error("Unable to import aerospike")
return
if str_to_bool(self.config["service_stats"]):
node_statistics = self._get_node_statistics()
self._publish_kv(node_statistics)
if str_to_bool(self.config["set_stats"]):
node_sets = self._get_node_sets()
self._publish_kv(node_sets)
if str_to_bool(self.config["latency_stats"]):
node_latency = self._get_node_latency()
self._publish_kv(node_latency)
if str_to_bool(self.config["namespace_stats"]):
node_namespace = self._get_node_namespace()
self._publish_kv(node_namespace)
def collect(self):
if ((not os.access(self.config['bin'], os.X_OK) or
(str_to_bool(self.config['use_sudo']) and
not os.access(self.config['sudo_cmd'], os.X_OK)))):
return
command = [self.config['bin'],
'--data', ",".join(self.config['vars']),
'--mrtg']
if str_to_bool(self.config['use_sudo']):
command.insert(0, self.config['sudo_cmd'])
p = subprocess.Popen(command,
stdout=subprocess.PIPE).communicate()[0][:-1]
for i, v in enumerate(p.split("\n")):
metric_name = self.config['vars'][i]
metric_value = int(v)
self.publish(metric_name, metric_value)
def get_output(self):
try:
command = [self.config['bin'], 'sourcestats']
if str_to_bool(self.config['use_sudo']):
command.insert(0, self.config['sudo_cmd'])
return subprocess.Popen(command,
stdout=subprocess.PIPE).communicate()[0]
except OSError:
return ""
def poll(self):
try:
command = [self.config["bin"], "-1"]
if str_to_bool(self.config["use_sudo"]):
command.insert(0, self.config["sudo_cmd"])
output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0]
except OSError:
output = ""
return output
def get_unbound_control_output(self):
try:
command = [self.config['bin'], ' stats']
if str_to_bool(self.config['use_sudo']):
command.insert(0, self.config['sudo_cmd'])
return subprocess.Popen(command,
stdout=subprocess.PIPE).communicate()[0]
except OSError:
self.log.exception("Unable to run %s", command)
return ""
def collect(self):
use_sudo = str_to_bool(self.config["use_sudo"])
if not os.access(self.config["bin"], os.X_OK) or (use_sudo and not os.access(self.config["sudo_cmd"], os.X_OK)):
return False
command = [self.config["bin"], "sensor"]
if use_sudo and getpass.getuser() != "root":
command.insert(0, self.config["sudo_cmd"])
p = Popen(command, stdout=PIPE).communicate()[0][:-1]
for i, v in enumerate(p.split("\n")):
data = v.split("|")
try:
# Complex keys are fun!
metric_name = data[0].strip()
metric_name = metric_name.replace(".", "_")
metric_name = metric_name.replace(" ", self.config["delimiter"])
metrics = []
# Each sensor line is a column seperated by a | with the
# following descriptions:
# 1. Sensor ID
# 2. Sensor Reading
# 3. Units
# 4. Status
# 5. Lower Non-Recoverable
# 6. Lower Critical
# 7. Lower Non-Critical
# 8. Upper Non-Critical
# 9. Upper Critical
# 10. Upper Non-Recoverable
if not self.config["thresholds"]:
metrics.append((metric_name, self.parse_value(data[1])))
else:
metrics.append((metric_name + ".Reading", self.parse_value(data[1])))
metrics.append((metric_name + ".Lower.NonRecoverable", self.parse_value(data[4])))
metrics.append((metric_name + ".Lower.Critical", self.parse_value(data[5])))
metrics.append((metric_name + ".Lower.NonCritical", self.parse_value(data[6])))
metrics.append((metric_name + ".Upper.NonCritical", self.parse_value(data[7])))
metrics.append((metric_name + ".Upper.Critical", self.parse_value(data[8])))
metrics.append((metric_name + ".Upper.NonRecoverable", self.parse_value(data[9])))
[self.publish(name, value) for (name, value) in metrics if value is not None]
except ValueError:
continue
except IndexError:
continue
return True
def collect(self):
"""
Collector cpu stats
"""
def cpu_time_list():
"""
get cpu time list
"""
statFile = open(self.PROC, "r")
timeList = statFile.readline().split(" ")[2:6]
for i in range(len(timeList)):
timeList[i] = int(timeList[i])
statFile.close()
return timeList
def cpu_delta_time(interval):
"""
Get before and after cpu times for usage calc
"""
pre_check = cpu_time_list()
time.sleep(interval)
post_check = cpu_time_list()
for i in range(len(pre_check)):
post_check[i] -= pre_check[i]
return post_check
if os.access(self.PROC, os.R_OK):
#If simple only return aggregate CPU% metric
if str_to_bool(self.config['simple']):
dt = cpu_delta_time(self.INTERVAL)
cpuPct = 100 - (dt[len(dt) - 1] * 100.00 / sum(dt))
self.publish('percent', str('%.4f' % cpuPct))
return True
results = {}
# Open file
file = open(self.PROC)
ncpus = -1 # dont want to count the 'cpu'(total) cpu.
for line in file:
if not line.startswith('cpu'):
continue
ncpus += 1
elements = line.split()
cpu = elements[0]
if cpu == 'cpu':
cpu = 'total'
elif not str_to_bool(self.config['percore']):
continue
results[cpu] = {}
if len(elements) >= 2:
results[cpu]['user'] = elements[1]
if len(elements) >= 3:
results[cpu]['nice'] = elements[2]
if len(elements) >= 4:
results[cpu]['system'] = elements[3]
if len(elements) >= 5:
results[cpu]['idle'] = elements[4]
if len(elements) >= 6:
results[cpu]['iowait'] = elements[5]
if len(elements) >= 7:
results[cpu]['irq'] = elements[6]
if len(elements) >= 8:
results[cpu]['softirq'] = elements[7]
if len(elements) >= 9:
results[cpu]['steal'] = elements[8]
if len(elements) >= 10:
results[cpu]['guest'] = elements[9]
if len(elements) >= 11:
results[cpu]['guest_nice'] = elements[10]
# Close File
file.close()
metrics = {}
for cpu in results.keys():
stats = results[cpu]
for s in stats.keys():
# Get Metric Name
metric_name = '.'.join([cpu, s])
# Get actual data
if (str_to_bool(self.config['normalize'])
and cpu == 'total' and ncpus > 0):
metrics[metric_name] = self.derivative(
metric_name,
long(stats[s]),
self.MAX_VALUES[s]) / ncpus
else:
metrics[metric_name] = self.derivative(
metric_name,
long(stats[s]),
self.MAX_VALUES[s])
# Check for a bug in xen where the idle time is doubled for guest
# See https://bugzilla.redhat.com/show_bug.cgi?id=624756
if self.config['xenfix'] is None or self.config['xenfix'] is True:
if os.path.isdir('/host_proc/xen'):
total = 0
for metric_name in metrics.keys():
if 'cpu0.' in metric_name:
total += int(metrics[metric_name])
if total > 110:
self.config['xenfix'] = True
for mname in metrics.keys():
if '.idle' in mname:
metrics[mname] = float(metrics[mname]) / 2
elif total > 0:
self.config['xenfix'] = False
else:
self.config['xenfix'] = False
# Publish Metric Derivative
for metric_name in metrics.keys():
self.publish(metric_name,
metrics[metric_name])
return True
else:
if not psutil:
self.log.error('Unable to import psutil')
self.log.error('No cpu metrics retrieved')
return None
cpu_time = psutil.cpu_times(True)
cpu_count = len(cpu_time)
total_time = psutil.cpu_times()
for i in range(0, len(cpu_time)):
metric_name = 'cpu' + str(i)
self.publish(metric_name + '.user',
self.derivative(metric_name + '.user',
cpu_time[i].user,
self.MAX_VALUES['user']))
if hasattr(cpu_time[i], 'nice'):
self.publish(metric_name + '.nice',
self.derivative(metric_name + '.nice',
cpu_time[i].nice,
self.MAX_VALUES['nice']))
self.publish(metric_name + '.system',
self.derivative(metric_name + '.system',
cpu_time[i].system,
self.MAX_VALUES['system']))
self.publish(metric_name + '.idle',
self.derivative(metric_name + '.idle',
cpu_time[i].idle,
self.MAX_VALUES['idle']))
metric_name = 'total'
self.publish(metric_name + '.user',
self.derivative(metric_name + '.user',
total_time.user,
self.MAX_VALUES['user'])
/ cpu_count)
if hasattr(total_time, 'nice'):
self.publish(metric_name + '.nice',
self.derivative(metric_name + '.nice',
total_time.nice,
self.MAX_VALUES['nice'])
/ cpu_count)
self.publish(metric_name + '.system',
self.derivative(metric_name + '.system',
total_time.system,
self.MAX_VALUES['system'])
/ cpu_count)
self.publish(metric_name + '.idle',
self.derivative(metric_name + '.idle',
total_time.idle,
self.MAX_VALUES['idle'])
/ cpu_count)
return True
return None
def collect(self):
if libvirt is None:
self.log.error('Unable to import libvirt')
return {}
conn = libvirt.openReadOnly(self.config['uri'])
for dom in [conn.lookupByID(n) for n in conn.listDomainsID()]:
if str_to_bool(self.config['sort_by_uuid']):
name = dom.UUIDString()
else:
name = dom.name()
# CPU stats
vcpus = dom.getCPUStats(True, 0)
totalcpu = 0
idx = 0
for vcpu in vcpus:
cputime = vcpu['cpu_time']
self.report_cpu_metric('cpu.%s.time' % idx, cputime, name)
idx += 1
totalcpu += cputime
self.report_cpu_metric('cpu.total.time', totalcpu, name)
# Disk stats
disks = self.get_disk_devices(dom)
accum = {}
for stat in self.blockStats.keys():
accum[stat] = 0
for disk in disks:
stats = dom.blockStats(disk)
for stat in self.blockStats.keys():
idx = self.blockStats[stat]
val = stats[idx]
accum[stat] += val
self.publish('block.%s.%s' % (disk, stat),
val,
instance=name)
for stat in self.blockStats.keys():
self.publish('block.total.%s' % stat,
accum[stat],
instance=name)
# Network stats
vifs = self.get_network_devices(dom)
accum = {}
for stat in self.vifStats.keys():
accum[stat] = 0
for vif in vifs:
stats = dom.interfaceStats(vif)
for stat in self.vifStats.keys():
idx = self.vifStats[stat]
val = stats[idx]
accum[stat] += val
self.publish('net.%s.%s' % (vif, stat), val, instance=name)
for stat in self.vifStats.keys():
self.publish('net.total.%s' % stat, accum[stat], instance=name)
# Memory stats
mem = dom.memoryStats()
self.publish('memory.nominal', mem['actual'] * 1024, instance=name)
self.publish('memory.rss', mem['rss'] * 1024, instance=name)
def collect(self):
"""Collect number values from db.serverStatus()"""
if pymongo is None:
self.log.error('Unable to import pymongo')
return
hosts = self.config.get('hosts')
# Convert a string config value to be an array
if isinstance(hosts, basestring):
hosts = [hosts]
# we need this for backwards compatibility
if 'host' in self.config:
hosts = [self.config['host']]
# convert network_timeout to integer
if self.config['network_timeout']:
self.config['network_timeout'] = int(
self.config['network_timeout'])
# convert collection_sample_rate to float
if self.config['collection_sample_rate']:
self.config['collection_sample_rate'] = float(
self.config['collection_sample_rate'])
# use auth if given
if 'user' in self.config:
user = self.config['user']
else:
user = None
if 'passwd' in self.config:
passwd = self.config['passwd']
else:
passwd = None
for host in hosts:
matches = re.search('((.+)\@)?(.+)?', host)
alias = matches.group(2)
host = matches.group(3)
if alias is None:
if len(hosts) == 1:
# one host only, no need to have a prefix
base_prefix = []
else:
base_prefix = [re.sub('[:\.]', '_', host)]
else:
base_prefix = [alias]
try:
# Ensure that the SSL option is a boolean.
if type(self.config['ssl']) is str:
self.config['ssl'] = str_to_bool(self.config['ssl'])
if ReadPreference is None:
conn = pymongo.Connection(
host,
network_timeout=self.config['network_timeout'],
ssl=self.config['ssl'],
slave_okay=True)
else:
conn = pymongo.Connection(
host,
network_timeout=self.config['network_timeout'],
ssl=self.config['ssl'],
read_preference=ReadPreference.SECONDARY,
)
except Exception, e:
self.log.error('Couldnt connect to mongodb: %s', e)
continue
# try auth
if user:
try:
conn.admin.authenticate(user, passwd)
except Exception, e:
self.log.error(
'User auth given, but could not autheticate' +
' with host: %s, err: %s' % (host, e))
return {}
def collect(self):
"""Collect number values from db.serverStatus()"""
if pymongo is None:
self.log.error('Unable to import pymongo')
return
hosts = self.config.get('hosts')
# Convert a string config value to be an array
if isinstance(hosts, basestring):
hosts = [hosts]
# we need this for backwards compatibility
if 'host' in self.config:
hosts = [self.config['host']]
# convert network_timeout to integer
if self.config['network_timeout']:
self.config['network_timeout'] = int(
self.config['network_timeout'])
# convert collection_sample_rate to float
if self.config['collection_sample_rate']:
self.config['collection_sample_rate'] = float(
self.config['collection_sample_rate'])
# use auth if given
if 'user' in self.config:
user = self.config['user']
else:
user = None
if 'passwd' in self.config:
passwd = self.config['passwd']
else:
passwd = None
for host in hosts:
matches = re.search('((.+)\@)?(.+)?', host)
alias = matches.group(2)
host = matches.group(3)
if alias is None:
if len(hosts) == 1:
# one host only, no need to have a prefix
base_prefix = []
else:
base_prefix = [re.sub('[:\.]', '_', host)]
else:
base_prefix = [alias]
try:
# Ensure that the SSL option is a boolean.
if type(self.config['ssl']) is str:
self.config['ssl'] = str_to_bool(self.config['ssl'])
if ReadPreference is None:
conn = pymongo.MongoClient(
host,
socketTimeoutMS=self.config['network_timeout'],
ssl=self.config['ssl'],
)
else:
conn = pymongo.MongoClient(
host,
socketTimeoutMS=self.config['network_timeout'],
ssl=self.config['ssl'],
read_preference=ReadPreference.SECONDARY,
)
except Exception as e:
self.log.error('Could not connect to mongodb: %s', e)
continue
# try auth
if user:
try:
conn.admin.authenticate(user, passwd)
except Exception as e:
self.log.error(
'User auth given, but could not autheticate' +
' with host: %s, err: %s' % (host, e))
return {}
data = conn.db.command('serverStatus')
self._publish_transformed(data, base_prefix)
if str_to_bool(self.config['simple']):
data = self._extract_simple_data(data)
if str_to_bool(self.config['replica']):
try:
replset_data = conn.admin.command('replSetGetStatus')
self._publish_replset(replset_data, base_prefix)
except pymongo.errors.OperationFailure as e:
self.log.error('error getting replica set status', e)
self._publish_dict_with_prefix(data, base_prefix)
db_name_filter = re.compile(self.config['databases'])
ignored_collections = re.compile(self.config['ignore_collections'])
sample_threshold = self.MAX_CRC32 * self.config[
'collection_sample_rate']
for db_name in conn.database_names():
if not db_name_filter.search(db_name):
continue
db_stats = conn[db_name].command('dbStats')
db_prefix = base_prefix + ['databases', db_name]
self._publish_dict_with_prefix(db_stats, db_prefix)
for collection_name in conn[db_name].collection_names():
if ignored_collections.search(collection_name):
continue
if (self.config['collection_sample_rate'] < 1
and (zlib.crc32(collection_name) & 0xffffffff) >
sample_threshold):
continue
collection_stats = conn[db_name].command(
'collstats', collection_name)
if str_to_bool(self.config['translate_collections']):
collection_name = collection_name.replace('.', '_')
collection_prefix = db_prefix + [collection_name]
self._publish_dict_with_prefix(collection_stats,
collection_prefix)
class MongoDBCollector(diamond.collector.Collector):
MAX_CRC32 = 4294967295
def __init__(self, *args, **kwargs):
self.__totals = {}
super(MongoDBCollector, self).__init__(*args, **kwargs)
def get_default_config_help(self):
config_help = super(MongoDBCollector, self).get_default_config_help()
config_help.update({
'hosts':
'Array of hostname(:port) elements to get metrics from'
'Set an alias by prefixing host:port with alias@',
'host':
'A single hostname(:port) to get metrics from'
' (can be used instead of hosts and overrides it)',
'user':
'******',
'passwd':
'Password for authenticated login (optional)',
'databases':
'A regex of which databases to gather metrics for.'
' Defaults to all databases.',
'ignore_collections':
'A regex of which collections to ignore.'
' MapReduce temporary collections (tmp.mr.*)'
' are ignored by default.',
'collection_sample_rate':
'Only send stats for a consistent subset '
'of collections. This is applied after collections are ignored via'
' ignore_collections Sampling uses crc32 so it is consistent across'
' replicas. Value between 0 and 1. Default is 1',
'network_timeout':
'Timeout for mongodb connection (in seconds).'
' There is no timeout by default.',
'simple':
'Only collect the same metrics as mongostat.',
'translate_collections':
'Translate dot (.) to underscores (_)'
' in collection names.',
'ssl':
'True to enable SSL connections to the MongoDB server.'
' Default is False'
})
return config_help
def get_default_config(self):
"""
Returns the default collector settings
"""
config = super(MongoDBCollector, self).get_default_config()
config.update({
'path': 'mongo',
'hosts': ['localhost'],
'user': None,
'passwd': None,
'databases': '.*',
'ignore_collections': '^tmp\.mr\.',
'network_timeout': None,
'simple': 'False',
'translate_collections': 'False',
'collection_sample_rate': 1,
'ssl': False
})
return config
def collect(self):
"""Collect number values from db.serverStatus()"""
if pymongo is None:
self.log.error('Unable to import pymongo')
return
hosts = self.config.get('hosts')
# Convert a string config value to be an array
if isinstance(hosts, basestring):
hosts = [hosts]
# we need this for backwards compatibility
if 'host' in self.config:
hosts = [self.config['host']]
# convert network_timeout to integer
if self.config['network_timeout']:
self.config['network_timeout'] = int(
self.config['network_timeout'])
# convert collection_sample_rate to float
if self.config['collection_sample_rate']:
self.config['collection_sample_rate'] = float(
self.config['collection_sample_rate'])
# use auth if given
if 'user' in self.config:
user = self.config['user']
else:
user = None
if 'passwd' in self.config:
passwd = self.config['passwd']
else:
passwd = None
for host in hosts:
matches = re.search('((.+)\@)?(.+)?', host)
alias = matches.group(2)
host = matches.group(3)
if alias is None:
if len(hosts) == 1:
# one host only, no need to have a prefix
base_prefix = []
else:
base_prefix = [re.sub('[:\.]', '_', host)]
else:
base_prefix = [alias]
try:
# Ensure that the SSL option is a boolean.
if type(self.config['ssl']) is str:
self.config['ssl'] = str_to_bool(self.config['ssl'])
if ReadPreference is None:
conn = pymongo.Connection(
host,
network_timeout=self.config['network_timeout'],
ssl=self.config['ssl'],
slave_okay=True)
else:
conn = pymongo.Connection(
host,
network_timeout=self.config['network_timeout'],
ssl=self.config['ssl'],
read_preference=ReadPreference.SECONDARY,
)
except Exception, e:
self.log.error('Couldnt connect to mongodb: %s', e)
continue
# try auth
if user:
try:
conn.admin.authenticate(user, passwd)
except Exception, e:
self.log.error(
'User auth given, but could not autheticate' +
' with host: %s, err: %s' % (host, e))
return {}
data = conn.db.command('serverStatus')
self._publish_transformed(data, base_prefix)
if str_to_bool(self.config['simple']):
data = self._extract_simple_data(data)
self._publish_dict_with_prefix(data, base_prefix)
db_name_filter = re.compile(self.config['databases'])
ignored_collections = re.compile(self.config['ignore_collections'])
sample_threshold = self.MAX_CRC32 * self.config[
'collection_sample_rate']
for db_name in conn.database_names():
if not db_name_filter.search(db_name):
continue
db_stats = conn[db_name].command('dbStats')
db_prefix = base_prefix + ['databases', db_name]
self._publish_dict_with_prefix(db_stats, db_prefix)
for collection_name in conn[db_name].collection_names():
if ignored_collections.search(collection_name):
continue
if (self.config['collection_sample_rate'] < 1
and (zlib.crc32(collection_name) & 0xffffffff) >
sample_threshold):
continue
collection_stats = conn[db_name].command(
'collstats', collection_name)
if str_to_bool(self.config['translate_collections']):
collection_name = collection_name.replace('.', '_')
collection_prefix = db_prefix + [collection_name]
self._publish_dict_with_prefix(collection_stats,
collection_prefix)
def collect_instance(self, alias, host, port):
result = self._get(host, port, '_nodes/_local/stats?all=true', 'nodes')
if not result:
return
metrics = {}
node = result['nodes'].keys()[0]
data = result['nodes'][node]
#
# http connections to ES
metrics['http.current'] = data['http']['current_open']
#
# indices
indices = data['indices']
metrics['indices.docs.count'] = indices['docs']['count']
metrics['indices.docs.deleted'] = indices['docs']['deleted']
metrics['indices.datastore.size'] = indices['store']['size_in_bytes']
transport = data['transport']
metrics['transport.rx.count'] = transport['rx_count']
metrics['transport.rx.size'] = transport['rx_size_in_bytes']
metrics['transport.tx.count'] = transport['tx_count']
metrics['transport.tx.size'] = transport['tx_size_in_bytes']
# elasticsearch < 0.90RC2
if 'cache' in indices:
cache = indices['cache']
self._add_metric(metrics, 'cache.bloom.size', cache,
['bloom_size_in_bytes'])
self._add_metric(metrics, 'cache.field.evictions', cache,
['field_evictions'])
self._add_metric(metrics, 'cache.field.size', cache,
['field_size_in_bytes'])
metrics['cache.filter.count'] = cache['filter_count']
metrics['cache.filter.evictions'] = cache['filter_evictions']
metrics['cache.filter.size'] = cache['filter_size_in_bytes']
self._add_metric(metrics, 'cache.id.size', cache,
['id_cache_size_in_bytes'])
# elasticsearch >= 0.90RC2
if 'filter_cache' in indices:
cache = indices['filter_cache']
metrics['cache.filter.evictions'] = cache['evictions']
metrics['cache.filter.size'] = cache['memory_size_in_bytes']
self._add_metric(metrics, 'cache.filter.count', cache, ['count'])
# elasticsearch >= 0.90RC2
if 'id_cache' in indices:
cache = indices['id_cache']
self._add_metric(metrics, 'cache.id.size', cache,
['memory_size_in_bytes'])
# elasticsearch >= 0.90
if 'fielddata' in indices:
fielddata = indices['fielddata']
self._add_metric(metrics, 'fielddata.size', fielddata,
['memory_size_in_bytes'])
self._add_metric(metrics, 'fielddata.evictions', fielddata,
['evictions'])
#
# process mem/cpu (may not be present, depending on access restrictions)
self._add_metric(metrics, 'process.cpu.percent', data,
['process', 'cpu', 'percent'])
self._add_metric(metrics, 'process.mem.resident', data,
['process', 'mem', 'resident_in_bytes'])
self._add_metric(metrics, 'process.mem.share', data,
['process', 'mem', 'share_in_bytes'])
self._add_metric(metrics, 'process.mem.virtual', data,
['process', 'mem', 'total_virtual_in_bytes'])
#
# filesystem (may not be present, depending on access restrictions)
if 'fs' in data and 'data' in data['fs'] and data['fs']['data']:
fs_data = data['fs']['data'][0]
self._add_metric(metrics, 'disk.reads.count', fs_data,
['disk_reads'])
self._add_metric(metrics, 'disk.reads.size', fs_data,
['disk_read_size_in_bytes'])
self._add_metric(metrics, 'disk.writes.count', fs_data,
['disk_writes'])
self._add_metric(metrics, 'disk.writes.size', fs_data,
['disk_write_size_in_bytes'])
#
# jvm
if 'jvm' in self.config['stats']:
jvm = data['jvm']
mem = jvm['mem']
for k in ('heap_used', 'heap_committed', 'non_heap_used',
'non_heap_committed'):
metrics['jvm.mem.%s' % k] = mem['%s_in_bytes' % k]
if 'heap_used_percent' in mem:
metrics['jvm.mem.heap_used_percent'] = mem['heap_used_percent']
for pool, d in mem['pools'].iteritems():
pool = pool.replace(' ', '_')
metrics['jvm.mem.pools.%s.used' % pool] = d['used_in_bytes']
metrics['jvm.mem.pools.%s.max' % pool] = d['max_in_bytes']
metrics['jvm.threads.count'] = jvm['threads']['count']
gc = jvm['gc']
collection_count = 0
collection_time_in_millis = 0
for collector, d in gc['collectors'].iteritems():
metrics['jvm.gc.collection.%s.count' %
collector] = d['collection_count']
collection_count += d['collection_count']
metrics['jvm.gc.collection.%s.time' %
collector] = d['collection_time_in_millis']
collection_time_in_millis += d['collection_time_in_millis']
# calculate the totals, as they're absent in elasticsearch > 0.90.10
if 'collection_count' in gc:
metrics['jvm.gc.collection.count'] = gc['collection_count']
else:
metrics['jvm.gc.collection.count'] = collection_count
k = 'collection_time_in_millis'
if k in gc:
metrics['jvm.gc.collection.time'] = gc[k]
else:
metrics['jvm.gc.collection.time'] = collection_time_in_millis
#
# thread_pool
if 'thread_pool' in self.config['stats']:
self._copy_two_level(metrics, 'thread_pool', data['thread_pool'])
#
# network
self._copy_two_level(metrics, 'network', data['network'])
#
# cluster (optional)
if str_to_bool(self.config['cluster']):
self.collect_instance_cluster_stats(host, port, metrics)
#
# indices (optional)
if 'indices' in self.config['stats']:
self.collect_instance_index_stats(host, port, metrics)
#
# all done, now publishing all metrics
for key in metrics:
full_key = key
if alias != '':
full_key = '%s.%s' % (alias, full_key)
self.publish(full_key, metrics[key])
def collect(self):
if not os.access(self.config['bin'], os.X_OK):
self.log.error("%s does not exist, or is not executable",
self.config['bin'])
return False
if ((str_to_bool(self.config['use_sudo'])
and not os.access(self.config['sudo_cmd'], os.X_OK))):
self.log.error("%s does not exist, or is not executable",
self.config['sudo_cmd'])
return False
p = subprocess.Popen(self.statcommand,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p.wait()
if p.returncode == 255:
self.statcommand = filter(lambda a: a != '--exact',
self.statcommand)
p = subprocess.Popen(self.statcommand,
stdout=subprocess.PIPE).communicate()[0][:-1]
columns = {
'conns': 2,
'inpkts': 3,
'outpkts': 4,
'inbytes': 5,
'outbytes': 6,
}
external = ""
backend = ""
for i, line in enumerate(p.split("\n")):
if i < 3:
continue
row = line.split()
if row[0] == "TCP" or row[0] == "UDP":
external = row[0] + "_" + string.replace(row[1], ".", "_")
backend = "total"
elif row[0] == "->":
backend = string.replace(row[1], ".", "_")
else:
continue
for metric, column in columns.iteritems():
metric_name = ".".join([external, backend, metric])
# metric_value = int(row[column])
value = row[column]
if value.endswith('K'):
metric_value = int(value[0:len(value) - 1]) * 1024
elif value.endswith('M'):
metric_value = (int(value[0:len(value) - 1]) * 1024 * 1024)
elif value.endswith('G'):
metric_value = (int(value[0:len(value) - 1]) * 1024 *
1024 * 1024)
else:
metric_value = float(value)
self.publish(metric_name, metric_value)
p = subprocess.Popen(self.concommand,
stdout=subprocess.PIPE).communicate()[0][:-1]
columns = {
'active': 4,
'inactive': 5,
}
external = ""
backend = ""
total = {}
for i, line in enumerate(p.split("\n")):
if i < 3:
continue
row = line.split()
if row[0] == "TCP" or row[0] == "UDP":
if total:
for metric, value in total.iteritems():
self.publish(".".join([external, "total", metric]),
value)
for k in columns.keys():
total[k] = 0.0
external = row[0] + "_" + string.replace(row[1], ".", "_")
continue
elif row[0] == "->":
backend = string.replace(row[1], ".", "_")
else:
continue
for metric, column in columns.iteritems():
metric_name = ".".join([external, backend, metric])
# metric_value = int(row[column])
value = row[column]
if value.endswith('K'):
metric_value = int(value[0:len(value) - 1]) * 1024
elif value.endswith('M'):
metric_value = int(value[0:len(value) - 1]) * 1024 * 1024
elif value.endswith('G'):
metric_value = (int(value[0:len(value) - 1]) * 1024 *
1024 * 1024)
else:
metric_value = float(value)
total[metric] += metric_value
self.publish(metric_name, metric_value)
if total:
for metric, value in total.iteritems():
self.publish(".".join([external, "total", metric]), value)
class TokuMXCollector(diamond.collector.Collector):
def __init__(self, *args, **kwargs):
self.__totals = {}
super(TokuMXCollector, self).__init__(*args, **kwargs)
def get_default_config_help(self):
config_help = super(TokuMXCollector, self).get_default_config_help()
config_help.update({
'hosts':
'Array of hostname(:port) elements to get metrics from'
'Set an alias by prefixing host:port with alias@',
'host':
'A single hostname(:port) to get metrics from'
' (can be used instead of hosts and overrides it)',
'user':
'******',
'passwd':
'Password for authenticated login (optional)',
'databases':
'A regex of which databases to gather metrics for.'
' Defaults to all databases.',
'ignore_collections':
'A regex of which collections to ignore.'
' MapReduce temporary collections (tmp.mr.*)'
' are ignored by default.',
'network_timeout':
'Timeout for mongodb connection (in seconds).'
' There is no timeout by default.',
'simple':
'Only collect the same metrics as mongostat.',
'translate_collections':
'Translate dot (.) to underscores (_)'
' in collection names.'
})
return config_help
def get_default_config(self):
"""
Returns the default collector settings
"""
config = super(TokuMXCollector, self).get_default_config()
config.update({
'path': 'mongo',
'hosts': ['localhost'],
'user': None,
'passwd': None,
'databases': '.*',
'ignore_collections': '^tmp\.mr\.',
'network_timeout': None,
'simple': 'False',
'translate_collections': 'False'
})
return config
def collect(self):
"""Collect number values from db.serverStatus() and db.engineStatus()"""
if pymongo is None:
self.log.error('Unable to import pymongo')
return
# we need this for backwards compatibility
if 'host' in self.config:
self.config['hosts'] = [self.config['host']]
# convert network_timeout to integer
if self.config['network_timeout']:
self.config['network_timeout'] = int(
self.config['network_timeout'])
# use auth if given
if 'user' in self.config:
user = self.config['user']
else:
user = None
if 'passwd' in self.config:
passwd = self.config['passwd']
else:
passwd = None
for host in self.config['hosts']:
if len(self.config['hosts']) == 1:
# one host only, no need to have a prefix
base_prefix = []
else:
matches = re.search('((.+)\@)?(.+)?', host)
alias = matches.group(2)
host = matches.group(3)
if alias is None:
base_prefix = [re.sub('[:\.]', '_', host)]
else:
base_prefix = [alias]
try:
if ReadPreference is None:
conn = pymongo.Connection(
host,
network_timeout=self.config['network_timeout'],
slave_okay=True)
else:
conn = pymongo.Connection(
host,
network_timeout=self.config['network_timeout'],
read_preference=ReadPreference.SECONDARY,
)
except Exception, e:
self.log.error('Couldnt connect to mongodb: %s', e)
continue
# try auth
if user:
try:
conn.admin.authenticate(user, passwd)
except Exception, e:
self.log.error(
'User auth given, but could not autheticate' +
' with host: %s, err: %s' % (host, e))
return {}
serverStatus = conn.db.command('serverStatus')
engineStatus = conn.db.command('engineStatus')
data = dict(serverStatus.items() + engineStatus.items())
self._publish_transformed(data, base_prefix)
if str_to_bool(self.config['simple']):
data = self._extract_simple_data(data)
self._publish_dict_with_prefix(data, base_prefix)
db_name_filter = re.compile(self.config['databases'])
ignored_collections = re.compile(self.config['ignore_collections'])
for db_name in conn.database_names():
if not db_name_filter.search(db_name):
continue
db_stats = conn[db_name].command('dbStats')
db_prefix = base_prefix + ['databases', db_name]
self._publish_dict_with_prefix(db_stats, db_prefix)
for collection_name in conn[db_name].collection_names():
if ignored_collections.search(collection_name):
continue
collection_stats = conn[db_name].command(
'collstats', collection_name)
if str_to_bool(self.config['translate_collections']):
collection_name = collection_name.replace('.', '_')
collection_prefix = db_prefix + [collection_name]
self._publish_dict_with_prefix(collection_stats,
collection_prefix)