class Netstat(plumd.Reader):
"""Plugin to measure various kernel metrics from /proc."""
defaults = {
'poll.interval': 10,
'proc_path': '/proc',
'proc_netstat_gauges': {
'TcpExt': ["SyncookiesSent", "SyncookiesRecv",
"SyncookiesFailed", "OutOfWindowIcmps", "TW", "TWRecycled",
"TWKilled", "DelayedACKLost", "ListenOverflows",
"ListenDrops", "TCPLostRetransmit", "TCPRenoFailures",
"TCPSackFailures", "TCPLossFailures", "TCPFastRetrans",
"TCPForwardRetrans", "TCPSlowStartRetrans", "TCPTimeouts",
"TCPLossProbes", "TCPLossProbeRecovery", "TCPAbortOnData",
"TCPAbortOnClose", "TCPAbortOnMemory", "TCPAbortOnTimeout",
"TCPAbortOnLinger", "TCPAbortFailed", "TCPMemoryPressures",
"TCPBacklogDrop", "TCPMinTTLDrop", "TCPTimeWaitOverflow",
"TCPFastOpenActive", "TCPFastOpenActiveFail",
"TCPFastOpenPassive", "TCPFastOpenPassiveFail",
"TCPFastOpenListenOverflow", "TCPSynRetrans"],
'IpExt': ["InNoRoutes", "InTruncatedPkts", "InBcastPkts",
"OutBcastPkts", "InOctets", "OutOctets", "InBcastOctets",
"OutBcastOctets", "InCsumErrors", "InNoECTPkts", "InECT1Pkts",
"InECT0Pkts", "InCEPkts"]
},
'proc_netstat_rates': {}
}
def __init__(self, log, config):
"""Plugin to measure various kernel metrics from /proc.
:param log: A logger
:type log: logging.RootLogger
:param config: a plumd.config.Conf configuration helper instance.
:type config: plumd.config.Conf
"""
super(Netstat, self).__init__(log, config)
self.config.defaults(Netstat.defaults)
self.calc = Differential()
self.proc_file = "{0}/net/netstat".format(config.get('proc_path'))
self.gauges = self.config.get('proc_netstat_gauges')
self.rates = self.config.get('proc_netstat_rates')
def poll(self):
"""Poll for kernel metrics under proc file net/netstat.
:rtype: ResultSet
"""
return plumd.ResultSet(self.check())
def check(self):
"""Return detailed network statitistics proc file net/netstat.
Note: ECT1Pkts and ECT0Pkts relate to ECT congestion notifications.
:rtype: plumd.Result
"""
# what metrics do we want to record?
result = plumd.Result("netstat")
# read the proc file
dat = None
with open(self.proc_file, 'r') as f:
dat = f.read().strip().split("\n")
# timestamp for Differential calculations
ts = time.time()
# split values into lists
dlist = deque([entry.split() for entry in dat])
# put lists into key: value dict
metrics = {}
while dlist:
headers = dlist.popleft()
values = dlist.popleft()
# { 'IpExt': {'InNoRoutes': 0, ...} } - [:-1] on IpExt: removes :
metrics[headers[0][:-1]] = dict(zip(headers, values))
# record gauges
for ext, mnames in self.gauges.items():
if ext not in metrics:
self.log.warn("netstat: unknown extension: {0}".format(ext))
del(self.gauges[ext])
continue
values = metrics[ext]
for mname in mnames:
if mname in values:
mstr = "{0}.{1}".format(ext, mname)
result.add(plumd.Int(mstr, values[mname]))
else:
self.log.warn("netstat: unknown metric {0}".format(mname))
self.gauges[ext].remove(mname)
# record rates
for ext, mnames in self.rates.items():
if ext not in metrics:
self.log.warn("netstat: unknown extension: {0}".format(ext))
del(self.rates[ext])
continue
values = metrics[ext]
for mname in mnames:
if mname in values:
mstr = "{0}.{1}".format(ext, mname)
mval = self.calc.per_second(mstr, float(values[mname]), ts)
result.add(plumd.Float(mstr, mval))
else:
self.log.warn("netstat: unknown metric {0}".format(mname))
self.rates[ext].remove(mname)
return [result]
class NetDev(plumd.Reader):
"""Plugin to measure various kernel metrics from /proc."""
rename = {
'packets': 'pkt',
'compressed': 'compr',
'multicast': 'mcast',
}
defaults = {
'poll.interval': 10,
'proc_path': '/proc',
'proc_netdev_re': "(virbr\d+)|(vnet\d+)"
}
def __init__(self, log, config):
"""Plugin to measure network interface metrics from proc file net/dev.
:param log: A logger
:type log: logging.RootLogger
:param config: a plumd.config.Conf configuration helper instance.
:type config: plumd.config.Conf
"""
super(NetDev, self).__init__(log, config)
config.defaults(NetDev.defaults)
self.proc_file = "{0}/net/dev".format(config.get('proc_path'))
self.dev_re = re.compile(config.get('proc_netdev_re'))
self.calc = Differential()
self.enabled = True
self.dev_cols = self.get_columns()
def get_columns(self):
cols = []
dat = get_file(self.proc_file)
lines = deque(dat.split("\n"))
nlines = len(lines)
# expect at least one network interface + 2 header lines
if nlines < 2:
msg = "NetDev: cannot parse {0}"
self.log.error(msg.format(self.proc_file))
self.enabled = False
return cols
# skip first line eg. Inter-| Receive
lines.popleft()
# next line has the header values
hdr_vals = deque(lines.popleft().split("|"))
# remove eg. face
hdr_vals.popleft()
if len(hdr_vals) != 2:
msg = "NetDev: cannot parse {0}"
self.log.error(msg.format(self.proc_files))
self.enabled = False
return cols
# values are receive then transmit
hdrs = dict(zip(["rx", "tx"], hdr_vals))
for pfx, metrics in hdrs.items():
for metric in metrics.split():
if metric in NetDev.rename:
metric = NetDev.rename[metric]
cols.append("{0}_{1}".format(pfx, metric))
return cols
def poll(self):
"""Return network interface metrics from proc file net/dev.
:rtype: ResultSet
"""
return plumd.ResultSet(self.check())
def check(self):
"""Return network interface metrics from proc file net/dev.
:rtype: list
"""
result = plumd.Result("netdev")
if not self.enabled:
return [result]
dat = {}
cols = self.dev_cols
regexp = self.dev_re
# read and process /proc/net/dev
with open(self.proc_file, 'r') as f:
lines = deque(f.read().strip().split("\n"))
# remove the two header lines - format fail :|
lines.popleft()
lines.popleft()
# for Differential
ts = time.time()
# now it's dev, metrics
for line in lines:
dat = line.split()
dev = dat[0].split(":")[0]
vals = dat[1:]
if regexp.match(dev):
continue
for key, val in dict(zip(cols, vals)).items():
mstr = "{0}.{1}".format(dev, key)
dval = self.calc.per_second(mstr, int(val), ts)
result.add(plumd.Float(mstr, dval))
return [result]
class DiskStats(plumd.Reader):
proc_colums = 14
"""Plugin to measure various kernel metrics from /proc."""
defaults = {
"poll.interval": 10,
"proc_path": "/proc",
"diskstats_dev_re": "dm-\d",
"diskstats_cols": [
"r",
"r_merge",
"r_sector",
"r_time",
"w",
"w_merge",
"w_sector",
"w_time",
"io_inprog",
"io_time",
"io_weighted_time",
],
}
def __init__(self, log, config):
"""Plugin to measure various kernel metrics from /proc.
:param log: A logger
:type log: logging.RootLogger
:param config: a plumd.config.Conf configuration helper instance.
:type config: plumd.config.Conf
"""
super(DiskStats, self).__init__(log, config)
config.defaults(DiskStats.defaults)
self.calc = Differential()
self.proc_file = "{0}/diskstats".format(config.get("proc_path"))
self.diskstats_dev_re = re.compile(config.get("diskstats_dev_re"))
self.diskstats_cols = self.config.get("diskstats_cols")
self.devices = [] # list of device names to record metrics for
self.enabled = True
# get list of available devices:
dat = get_file_map(self.proc_file, 2, 0)
# key is the device name, exclude ones matching the re
for key, val in dat.items():
if not self.diskstats_dev_re.match(key):
self.devices.append(key)
# check format of proc file
ncols = len(get_file(self.proc_file).split("\n")[0].split())
self.enabled = ncols == DiskStats.proc_colums
if not self.enabled:
msg = "DiskStats: invalid format: {0} has {1} cols, not {2}"
self.log.error(msg.format(self.proc_file, ncols, DiskStats.proc_colums))
def poll(self):
"""Poll for kernel metrics under /proc.
:rtype: ResultSet
"""
return plumd.ResultSet(self.check())
def check(self):
"""Return disk io metrics from proc file diskstats.
:rtype: plumd.Result
"""
# times in ms
cols = self.diskstats_cols
result = plumd.Result("diskstats")
if not self.enabled:
return [result]
dat = {}
# read and process /proc/diskstats
dat = get_file_map(self.proc_file, 2, 0)
ts = time.time()
for dev in self.devices:
if dev not in dat:
continue
for mname in cols:
mval = float(dat[dev].popleft())
mstr = "{0}.{1}".format(dev, mname)
dval = self.calc.per_second(mstr, mval, ts)
result.add(plumd.Float(mstr, dval))
return [result]
class Redis(plumd.Reader):
"""Plugin to record redis metrics."""
# default config values
defaults = {
'poll.interval': 10,
'gauges': [
"aof_current_rewrite_time_sec",
"aof_enabled",
"aof_last_rewrite_time_sec",
"aof_rewrite_in_progress",
"aof_rewrite_scheduled",
"blocked_clients",
"client_biggest_input_buf",
"client_longest_output_list",
"connected_clients",
"connected_slaves",
"evicted_keys",
"expired_keys",
"instantaneous_input_kbps",
"instantaneous_ops_per_sec",
"instantaneous_output_kbps",
"keyspace_hits",
"keyspace_misses",
"latest_fork_usec",
"loading",
"master_repl_offset",
"mem_fragmentation_ratio",
"pubsub_channels",
"pubsub_patterns",
"rdb_bgsave_in_progress",
"rdb_changes_since_last_save",
"rdb_current_bgsave_time_sec",
"rdb_last_bgsave_time_sec",
"rdb_last_save_time",
"rejected_connections",
"repl_backlog_active",
"repl_backlog_first_byte_offset",
"repl_backlog_histlen",
"repl_backlog_size",
"sync_full",
"sync_partial_err",
"sync_partial_ok",
"total_commands_processed",
"total_connections_received",
"total_net_input_bytes",
"total_net_output_bytes",
"uptime_in_days",
"uptime_in_seconds",
"used_cpu_sys",
"used_cpu_sys_children",
"used_cpu_user",
"used_cpu_user_children",
"used_memory",
"used_memory_lua",
"used_memory_peak",
"used_memory_rss",
"master_last_io_seconds_ago",
"master_sync_in_progress",
"slave_repl_offset",
"slave_priority",
"slave_read_only",
"connected_slaves",
"master_repl_offset",
"repl_backlog_active",
"repl_backlog_size",
"repl_backlog_first_byte_offset",
"repl_backlog_histlen"
"connected_slaves"
],
'rates': [],
'configs': [
'maxmemory'
],
'keys': {
# 'type': { metric_prefix: [key_prefix*, ...] }
'lists': {},
'zsets': {},
'sets': {},
'hlls': {}
},
'addr': '127.0.0.1:6379',
'addr_type': 'inet',
'timeout': 10
}
def __init__(self, log, config):
"""Plugin to record redis metrics.
:param log: A logger
:type log: logging.RootLogger
:param config: a plumd.config.Conf configuration helper instance.
:type config: plumd.config.Conf
"""
super(Redis, self).__init__(log, config)
self.config.defaults(Redis.defaults)
# metrics to record
self.gauges = self.config.get('gauges')
self.rates = self.config.get('rates')
self.configs = self.config.get('configs')
self.keys = self.config.get('keys')
# Redis connection - either unix socket or tcp
addr = self.config.get('addr')
addr_type = self.config.get('addr_type').lower()
if addr_type == "unix":
sfamily = socket.AF_UNIX
elif addr_type == "inet":
try:
host, port = addr.split(":")
except AttributeError as e:
msg = "Redis: invalid address: {0}, (host:port)"
raise plumd.ConfigError(msg.format(addr))
addr = (host, int(port))
sfamily = socket.AF_INET
else:
msg = "Redis: unsupported connection type: {0} (unix, inet)"
raise plumd.ConfigError(msg.format(addr_type))
timeout = config.get('timeout')
self.client = RedisClient(self.log, addr, sfamily, timeout)
self.calc = Differential()
def poll(self):
"""Query Redis for metrics.
:rtype: ResultSet
"""
# catch exceptions - simply skip the poll on error
try:
result = plumd.Result("redis")
# config values
self.record_configs(result)
# key sizes
self.record_sizes(result)
# get server metrics
stats = self.client.info()
# record gauges, rates
self.record_metrics(stats, result)
# replication, if any slaves are connected
if "slave0" in stats:
self.record_slaves(stats, result)
# db metrics, maxmem
self.record_dbs(stats, result)
# record lists, zsets, sets and hll sizes
self.record_sizes(result)
# and finally command stats - if available
self.record_cmdstats(result)
except RedisError as e:
msg = "Redis: exception during poll: {0}"
self.log.error(msg.format(e))
return plumd.ResultSet([result])
def record_cmdstats(self, result):
"""Record the stats from info commandstats.
:param result: A result object to add metrics to
:type result: ResultSet
"""
ts = time.time()
name = self.name
infos = self.client.info("commandstats")
for key in sorted(infos.keys()):
vals = infos[key].split(",")
cstat, cname = key.split("_")
for val in vals:
mname, mval = val.split("=")
metric = "{0}.{1}.{2}.{3}".format(name, cstat, cname, mname)
result.add(plumd.Float(metric, mval))
def record_metrics(self, stats, result):
"""Record the configured gauges and metrics
:param stats: Dictionary returned from info command
:type stats: dict
:param result: A result object to add metrics to
:type result: ResultSet
"""
ts = time.time()
name = self.name
# record gauges
for stat in self.gauges:
if stat in stats:
mname = "{0}.{1}".format(name, stat)
result.add(plumd.Float(mname, stats[stat]))
# record rates
for stat in self.rates:
if stat in stats:
mname = "{0}.{1}".format(name, stat)
mval = self.calc.per_second(mname, float(stats[stat]), ts)
result.add(plumd.Float(mname, mval))
def record_dbs(self, stats, result):
"""Record per database metrics into result.
:param stats: Dictionary returned from info command
:type stats: dict
:param result: A result object to add metrics to
:type result: ResultSet
"""
# db0:keys=1,expires=0,avg_ttl=0
name = self.name
db_fmt = "db{0}"
metric_fmt = "{0}.db.{1}.{2}"
for i in xrange(0, len(stats.keys())):
dbname = db_fmt.format(i)
if dbname not in stats:
break
try:
vals = stats[dbname].split(",")
dbmetrics = dict((k, v)
for k, v in (v.split('=') for v in vals))
for k, v in dbmetrics.items():
metric_str = metric_fmt.format(name, i, k)
result.add(plumd.Int(metric_str, v))
except KeyError as E:
self.log.error("Redis: invalid db entry: {0}".format(dbname))
def record_slaves(self, stats, result):
"""Record slave metrics into result.
:param stats: A dictionary returned from info command
:type stats: dict
:param result: A ResultSet object to add metrics to
:type result: ResultSet
"""
# slave0:ip=127.0.0.1,port=6399,state=online,offset=239,lag=1
name = self.name
slave_str = "slave{0}"
moffstr = 'master_repl_offset'
moffset = 0
try:
moffset = int(stats[moffstr])
except(TypeError, KeyError) as e:
self.log.error("Redis: no {0} value".format(moffstr))
# for each slave entry
for i in xrange(0, len(stats.keys())):
sname = slave_str.format(i)
if sname not in stats:
break
try:
vals = stats[sname].split(",")
smetrics = dict((k, v)
for k, v in (v.split('=') for v in vals))
sip = smetrics['ip'].replace(".", "_")
smname = "{0}_{1}".format(sip, smetrics['port'])
# record offset and lag
mname = "{0}.slave.{1}.offset".format(name, smname)
soffset = moffset - int(smetrics['offset'])
result.add(plumd.Int(mname, soffset))
mname = "{0}.slave.{1}.lag".format(name, sname)
result.add(plumd.Int(mname, smetrics['lag']))
# if slave is online set online = 1, otherwise 0
sonline = 1 if smetrics['state'] == "online" else 0
mname = "{0}.slave.{1}.online".format(name, sname)
result.add(plumd.Int(mname, sonline))
except(TypeError, KeyError, ValueError) as e:
self.log.error("Redis: invalid slave entry: {0}".format(sname))
def record_configs(self, result):
"""Record the configured configuration values.
:param result: A ResultSet to record max mem to.
:type result: plumd.ResultSet
"""
configs = self.configs
if not configs:
return
name = self.name
for config in self.client.config_get_multi(configs):
for key, val in config.items():
mstr = "{0}.configs.{1}".format(name, key)
result.add(plumd.Float(mstr, val))
def record_sizes(self, result):
"""For each type of key (list, zset, set, hyperloglog)
scan for a list of keys matching the prefix and record the
total number of items for the matching prefix.
:param result: A ResultSet to record into.
:type result: plumd.ResultSet
"""
if not self.keys:
return
keys = self.config.get("keys")
if "lists" in keys:
self.record_lists(keys['lists'], result)
if "zsets" in keys:
self.record_zsets(keys['zsets'], result)
if "sets" in keys:
self.record_sets(keys['sets'], result)
if "hlls" in keys:
self.record_hlls(keys['hlls'], result)
def record_lists(self, lconfig, result):
"""Record the total length of the configured lists:
eg. lconfig: {"metric_name": [ "list*", "of*", "globs*"]}
:param lconfig: A dict of metric name => globs
:type lconfig: dict
:param result: A ResultSet to record into.
:type result: plumd.ResultSet
"""
name = self.name
for mprefix, kprefixes in lconfig.items():
for prefix in kprefixes:
# get the total for this prefix
total = self.client.llen_multi(self.client.scan(prefix))
mstr = "{0}.sizes.lists.{1}".format(name, mprefix)
result.add(plumd.Int(mstr, total))
def record_zsets(self, zconfig, result):
"""Record the total length of the configured zsets:
eg. zconfig: {"metric_name": [ "list*", "of*", "globs*"]}
:param zconfig: A dict of metric name => globs
:type zconfig: dict
:param result: A ResultSet to record into.
:type result: plumd.ResultSet
"""
name = self.name
for mprefix, kprefixes in zconfig.items():
for prefix in kprefixes:
# get the total for this prefix
total = self.client.zcard_multi(self.client.scan(prefix))
mstr = "{0}.sizes.zset.{1}".format(name, mprefix)
result.add(plumd.Int(mstr, total))
def record_sets(self, sconfig, result):
"""Record the total length of the configured zsets:
eg. sconfig: {"metric_name": [ "list*", "of*", "globs*"]}
:param sconfig: A dict of metric name => globs
:type sconfig: dict
:param result: A ResultSet to record into.
:type result: plumd.ResultSet
"""
name = self.name
for mprefix, kprefixes in sconfig.items():
for prefix in kprefixes:
# get the total for this prefix
total = self.client.scard_multi(self.client.scan(prefix))
mstr = "{0}.sizes.set.{1}".format(name, mprefix)
result.add(plumd.Int(mstr, total))
def record_hlls(self, hllconfig, result):
"""Record the total length of the configured hlls:
eg. sconfig: {"metric_name": [ "list*", "of*", "globs*"]}
:param hllconfig: A dict of metric name => globs
:type hllconfig: dict
:param result: A ResultSet to record into.
:type result: plumd.ResultSet
"""
name = self.name
for mprefix, kprefixes in hllconfig.items():
for prefix in kprefixes:
# get the total for this prefix
total = self.client.pfcount_multi(self.client.scan(prefix))
mstr = "{0}.sizes.hll.{1}".format(name, mprefix)
result.add(plumd.Int(mstr, total))
class NetSnmp(plumd.Reader):
"""Plugin to measure various kernel metrics from /proc."""
defaults = {
"poll.interval": 10,
"proc_path": "/proc",
"proc_netsnmp_gauges": {
"Ip": [
"Forwarding",
"InReceives",
"InHdrErrors",
"InAddrErrors",
"ForwDatagrams",
"InUnknownProtos",
"InDiscards",
"InDelivers",
"OutRequests",
"OutDiscards",
"ReasmTimeout",
"ReasmReqds",
"ReasmOKs",
"ReasmFails",
"FragOKs",
"FragFails",
"FragCreates",
],
"Icmp": [
"InMsgs",
"InErrors",
"InCsumErrors",
"InDestUnreachs",
"InTimeExcds",
"InParmProbs",
"InSrcQuenchs",
"InRedirects",
"InEchos",
"InEchoReps",
"OutMsgs",
"OutErrors",
"OutDestUnreachs",
"OutTimeExcds",
"OutParmProbs",
"OutSrcQuenchs",
"OutRedirects",
"OutEchos",
"OutEchoReps",
],
"IcmpMsg": ["OutType3"],
"Tcp": [
"MaxConn",
"ActiveOpens",
"PassiveOpens",
"AttemptFails",
"EstabResets",
"CurrEstab",
"InSegs",
"OutSegs",
"RetransSegs",
"InErrs",
"OutRsts",
"InCsumErrors",
],
"Udp": [
"InDatagrams",
"NoPorts",
"InErrors",
"OutDatagrams",
"RcvbufErrors",
"SndbufErrors",
"InCsumErrors",
],
},
"proc_netsnmp_rates": {},
}
def __init__(self, log, config):
"""Plugin to measure various kernel metrics from /proc.
:param log: A logger
:type log: logging.RootLogger
:param config: a plumd.config.Conf configuration helper instance.
:type config: plumd.config.Conf
"""
super(NetSnmp, self).__init__(log, config)
self.config.defaults(NetSnmp.defaults)
self.calc = Differential()
self.proc_file = "{0}/net/snmp".format(config.get("proc_path"))
self.filter = Filter()
self.gauges = self.config.get("proc_netsnmp_gauges")
self.rates = self.config.get("proc_netsnmp_rates")
def poll(self):
"""Poll for kernel metrics under /proc.
:rtype: ResultSet
"""
return plumd.ResultSet(self.check())
def check(self):
"""Return network protocol metrics from proc file net/snmp.
Add entries to the configuration value 'skip_proc_net_snmp' to skip
metrics.
Add entries to the configuration value 'net_snmp_items' to match the
format/order of the proc file net/snmp entries on the system.
:rtype: plumd.Result
"""
result = plumd.Result("netsnmp")
# read the proc file
dat = []
with open(self.proc_file, "r") as f:
dat = f.read().strip().split("\n")
# timestamp for Differential calculations
ts = time.time()
# split values into lists
dlist = deque([entry.split() for entry in dat])
# put lists into key: value dict
metrics = {}
while dlist:
headers = dlist.popleft()
values = dlist.popleft()
# { 'IpExt': {'InNoRoutes': 0, ...} } - [:-1] on IpExt: removes :
metrics[headers[0][:-1]] = dict(zip(headers, values))
# record gauges
for proto, mnames in self.gauges.items():
if proto not in metrics:
self.log.warn("netsnmp: unknown protocol: {0}".format(proto))
del (self.gauges[proto])
continue
values = metrics[proto]
for mname in mnames:
if mname in values:
mstr = "{0}.{1}".format(proto, mname)
result.add(plumd.Int(mstr, values[mname]))
else:
self.log.warn("netstat: unknown metric {0}".format(mname))
self.gauges[proto].remove(mname)
continue
# record rates
for proto, mnames in self.rates.items():
if proto not in metrics:
self.log.warn("netsnmp: unknown protocol: {0}".format(proto))
del (self.gauges[proto])
continue
values = metrics[proto]
for mname in mnames:
if mname in values:
mstr = "{0}.{1}".format(proto, mname)
mval = self.calc.per_second(mstr, int(values[mname]), ts)
result.add(plumd.Int(mstr, mval))
else:
self.log.warn("netstat: unknown metric {0}".format(mname))
self.rates[proto].remove(mname)
continue
return [result]
class Stat(plumd.Reader):
"""Class to read metrics from /proc/stat"""
defaults = {
"proc_stat_gauges": ["procs_running", "procs_blocked"],
"proc_stat_rates": ["intr", "ctxt", "softirq"],
"cpu_metrics": ["user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", "guest", "guest_nice"],
"per_cpu": False,
}
def __init__(self, log, config):
"""Plugin to measure various kernel metrics from /proc/stat
:param log: A logger
:type log: logging.RootLogger
:param config: a plumd.config.Conf configuration helper instance.
:type config: plumd.config.Conf
"""
super(Stat, self).__init__(log, config)
self.config.defaults(Stat.defaults)
self.calc = Differential()
self.proc_file = "{0}/stat".format(config.get("proc_path"))
self.per_cpu = self.config.get("per_cpu")
self.cpu_metrics = self.config.get("cpu_metrics")
self.gauges = self.config.get("proc_stat_gauges")
self.rates = self.config.get("proc_stat_rates")
def poll(self):
"""Return cpu utilization and process metrics from proc file stat.
:rtype: plumd.ResultSet
"""
return plumd.ResultSet(self.check())
def check(self):
"""Return cpu utilization and process metrics from proc file stat.
:rtype: collections.deque
"""
results = deque()
result = plumd.Result("stat")
dat = get_file_map(self.proc_file, 0, 0)
ts = time.time()
# record gauges
for i, metric in enumerate(self.gauges):
if metric not in dat:
self.log.warn("stat: unknown metric {0}".format(metric))
del (self.gauges[i])
continue
result.add(plumd.Int(metric, dat[metric][0]))
# record rates
for i, metric in enumerate(self.rates):
if metric not in dat:
self.log.warn("stat: unknown metric {0}".format(metric))
del (self.rates[i])
continue
mval = self.calc.per_second(metric, float(dat[metric][0]), ts)
result.add(plumd.Int(metric, mval))
# record cpu
if "cpu" in dat:
results.append(self.proc_stat_cpu("cpu", "cpu", dat["cpu"], ts))
# record each cpu if configured
if self.per_cpu:
for i in xrange(0, len(dat)):
mstr = "cpu{0}".format(i)
if mstr not in dat:
break
results.append(self.proc_stat_cpu("cpus", mstr, dat[mstr]))
results.append(result)
return results
def proc_stat_cpu(self, rname, key, val, ts):
"""Return cpu utilization metrics in percentage.
:param rname: The Result name (eg. cpu or cpus)
:type rname: str
:param val: A deque populated with the metric values from stat
:type val: deque
:rtype: list
"""
result = plumd.Result(rname)
cpu = self.cpu_metrics
for map_val in cpu:
if not val:
break
mstr = "{0}_{1}".format(key, map_val)
mval = self.calc.per_second(mstr, float(val.popleft()), ts)
result.add(plumd.Float(mstr, mval))
return result
class VmStat(plumd.Reader):
"""Plugin to measure various kernel metrics from /proc/vmstat."""
defaults = {
'poll.interval': 10,
'proc_path': '/proc',
# awk '{ print "\"" $1 "\"," }' /proc/vmstat
'proc_vmstat_gauges': ["nr_free_pages", "nr_dirtied", "nr_written",
"numa_hit", "numa_miss", "numa_foreign",
"numa_interleave", "numa_local", "numa_other",
"pgpgin", "pgpgout", "pswpin", "pswpout",
"pgalloc_dma", "pgalloc_dma32", "pgalloc_normal",
"pgalloc_movable", "pgfree", "pgactivate",
"pgdeactivate", "pgfault", "pgmajfault",
"slabs_scanned", "kswapd_inodesteal",
"kswapd_low_wmark_hit_quickly",
"kswapd_high_wmark_hit_quickly",
"drop_pagecache", "drop_slab",
"numa_pte_updates", "numa_huge_pte_updates",
"numa_hint_faults", "numa_hint_faults_local",
"numa_pages_migrated", "pgmigrate_success",
"pgmigrate_fail", "compact_migrate_scanned",
"compact_free_scanned", "compact_isolated",
"compact_stall", "compact_fail",
"compact_success"],
'proc_vmstat_rates': []
}
def __init__(self, log, config):
"""Plugin to measure various kernel metrics from /proc/vmstat.
:param log: A logger
:type log: logging.RootLogger
:param config: a plumd.config.Conf configuration helper instance.
:type config: plumd.config.Conf
"""
super(VmStat, self).__init__(log, config)
self.config.defaults(VmStat.defaults)
self.calc = Differential()
self.proc_file = "{0}/vmstat".format(config.get('proc_path'))
self.gauges = self.config.get('proc_vmstat_gauges')
self.rates = self.config.get('proc_vmstat_rates')
def poll(self):
"""Poll for kernel metrics under proc file vmstat.
:rtype: ResultSet
"""
return plumd.ResultSet(self.check())
def check(self):
"""Return metrics from /proc/vmstat.
:rtype: plumd.Result
"""
# what metrics do we want to record?
result = plumd.Result("vmstat")
# read the proc file
dat = {}
with open(self.proc_file, 'r') as f:
# get a list of values: metric, val, metric, val, etc
vals = f.read().strip().split()
# every second item starting at 0, every second item starting at 1
# take that and put into a dict
dat = dict(zip(vals[0::2], vals[1::2]))
# timestamp for Differential calculations
ts = time.time()
# now, we have a list of items to record, just need to record them
for i, metric in enumerate(self.gauges):
if metric in dat:
#mval = self.calc.per_second(mstr, int(values[mname]), ts)
result.add(plumd.Int(metric, dat[metric]))
else:
self.log.warn("vmstat: unknown metric {0}".format(metric))
del(self.gauges[i])
# and the rates as well, if any
for i, metric in enumerate(self.rates):
if metric in dat:
mval = self.calc.per_second(metric, int(dat[metric]), ts)
result.add(plumd.Float(metric, mval))
else:
self.log.warn("vmstat: unknown metric {0}".format(metric))
del(self.rates[i])
return [result]
class Memcache(plumd.Reader):
"""Plugin to record memcache stats."""
STAT_CMD = "stats\n"
RECV_SIZE = 4096
# default config values
defaults = {
"poll.interval": 10,
"gauges": [
"auth_cmds",
"auth_errors",
"bytes",
"bytes_read",
"bytes_written",
"cas_badval",
"cas_hits",
"cas_misses",
"cmd_flush",
"cmd_get",
"cmd_set",
"cmd_touch",
"connection_structures",
"conn_yields",
"curr_connections",
"curr_items",
"decr_hits",
"decr_misses",
"delete_hits",
"delete_misses",
"evicted_unfetched",
"evictions",
"expired_unfetched",
"get_hits",
"get_misses",
"hash_bytes",
"hash_is_expanding",
"hash_power_level",
"incr_hits",
"incr_misses",
"limit_maxbytes",
"listen_disabled_num",
"reclaimed",
"reserved_fds",
"rusage_system",
"rusage_user",
"threads",
"total_connections",
"total_items",
"touch_hits",
"touch_misses",
"uptime",
],
"rates": [],
"host": "127.0.0.1", # memcache server hostname/ip
"port": 11211, # memcache server port
"socket": None, # use tcp or unix domain socket
"timeout": 10, # connection timeouts
}
def __init__(self, log, config):
"""Plugin to record memcache stats.
:param log: A logger
:type log: logging.RootLogger
:param config: a plumd.config.Conf configuration helper instance.
:type config: plumd.config.Conf
"""
super(Memcache, self).__init__(log, config)
self.config.defaults(Memcache.defaults)
# metrics to record
self.gauges = self.config.get("gauges")
self.rates = self.config.get("rates")
# memcached connection - either unix socket or tcp
spath = self.config.get("socket")
if spath is None or not os.path.exists(spath):
host = self.config.get("host")
port = self.config.get("port")
self.addr = (host, port)
self.sfamily = socket.AF_INET
else:
self.addr = spath
self.sfamily = socket.AF_UNIX
self.socket = None
self.timeout = config.get("timeout")
self.calc = Differential()
def poll(self):
"""Query memcache for stats.
:rtype: ResultSet
"""
result = plumd.Result("memcache")
name = self.name
stats = self.get_stats()
ts = time.time()
# record gauges
for stat in self.gauges:
if stat in stats:
mname = "{0}.{1}".format(name, stat)
result.add(plumd.Float(mname, stats[stat]))
# record rates
for stat in self.rates:
if stat in stats:
mname = "{0}.{1}".format(name, stat)
mval = self.calc.per_second(mname, float(stats[stat]), ts)
result.add(plumd.Float(mname, mval))
return plumd.ResultSet([result])
def get_stats(self):
"""Request and read stats from memcache socket."""
stats = {}
if not self.socket and not self.connect():
return {}
try:
if PY3:
self.socket.sendall(bytes(Memcache.STAT_CMD, "utf8"))
else:
self.socket.sendall(Memcache.STAT_CMD)
st_str = self.socket.recv(Memcache.RECV_SIZE)
# self.log.debug("Memcached: read: {0}".format(st_str))
for line in st_str.split("\n"):
vals = line.split()
if len(vals) != 3:
continue
stype, sname, sval = vals
if stype == "STAT":
msg = "Memcached: {0} = {1}"
# self.log.debug(msg.format(sname, sval))
stats[sname] = sval
except Exception as e:
msg = "Memcached: {0}: poll: exception: {1}"
self.log.error(msg.format(self.addr, e))
self.disconnect()
return stats
def connect(self):
"""Connect to memcached, returns True if sucessful, False otherwise.
:rtype: bool
"""
# self.log.debug("Memcached: connecting: {0}".format(self.addr))
if self.socket:
self.disconnect()
try:
# create the socket
self.socket = socket.socket(self.sfamily, socket.SOCK_STREAM)
# set timeout for socket operations
self.socket.settimeout(self.timeout)
# connect
self.socket.connect(self.addr)
# log and return
msg = "Memcached: connected: {0}"
self.log.info(msg.format(self.addr))
return True
except Exception as e:
# log exception, ensure cleanup is done (disconnect)
msg = "Memcached: {0}: connect: excception: {1}"
self.log.error(msg.format(self.addr, e))
return False
return False
def disconnect(self):
"""Severe the memcached connection."""
if self.socket:
try:
self.socket.close()
self.socket = None
except Exception as e:
msg = "Memcached: dicsonnect: exception {0}".format(e)
self.log.error(msg)
self.socket = None