def __init__(self, config=None):
"""
Create a new instance of the GmetricHandler class
"""
# Initialize Handler
Handler.__init__(self, config)
# Initialize Data
self.socket = None
# Initialize Options
self.host = self.config['host']
self.port = int(self.config['port'])
self.protocol = self.config['protocol']
if not self.protocol:
self.protocol = 'udp'
# Initialize
self.gmetric = gmetric.Gmetric(self.host, self.port, self.protocol)
def __init__(self, config=None):
"""
Create a new instance of the GmetricHandler class
"""
# Initialize Handler
Handler.__init__(self, config)
if gmetric is None:
logging.error("Failed to load gmetric module")
return
# Initialize Data
self.socket = None
# Initialize Options
self.host = self.config['host']
self.port = int(self.config['port'])
self.protocol = self.config['protocol']
if not self.protocol:
self.protocol = 'udp'
# Initialize
self.gmetric = gmetric.Gmetric(self.host, self.port, self.protocol)
def flush(self):
ts = int(time.time())
stats = 0
if self.transport == 'graphite':
stat_string = ''
else:
g = gmetric.Gmetric(self.ganglia_host, self.ganglia_port,
self.ganglia_protocol)
for k, v in self.counters.items():
v = float(v)
v = v if self.no_aggregate_counters else v / (self.flush_interval /
1000)
if self.debug:
print "Sending %s => count=%s" % (k, v)
if self.transport == 'graphite':
msg = '%s.%s %s %s\n' % (self.counters_prefix, k, v, ts)
stat_string += msg
else:
# We put counters in _counters group. Underscore is to make sure counters show up
# first in the GUI. Change below if you disagree
g.send(k, v, "double", "count", "both", 60, self.dmax,
"_counters", self.ganglia_spoof_host)
self.counters[k] = 0
stats += 1
for k, v in self.timers.items():
if len(v) > 0:
# Sort all the received values. We need it to extract percentiles
v.sort()
count = len(v)
min = v[0]
max = v[-1]
mean = min
max_threshold = max
if count > 1:
thresh_index = int((self.pct_threshold / 100.0) * count)
max_threshold = v[thresh_index - 1]
total = sum(v)
mean = total / count
self.timers[k] = []
if self.debug:
print "Sending %s ====> lower=%s, mean=%s, upper=%s, %dpct=%s, count=%s" % (
k, min, mean, max, self.pct_threshold, max_threshold,
count)
if self.transport == 'graphite':
stat_string += TIMER_MSG % {
'prefix': self.timers_prefix,
'key': k,
'mean': mean,
'max': max,
'min': min,
'count': count,
'max_threshold': max_threshold,
'pct_threshold': self.pct_threshold,
'ts': ts,
}
else:
# What group should these metrics be in. For the time being we'll set it to the name of the key
group = k
g.send(k + "_lower", min, "double", "time", "both", 60,
self.dmax, group, self.ganglia_spoof_host)
g.send(k + "_mean", mean, "double", "time", "both", 60,
self.dmax, group, self.ganglia_spoof_host)
g.send(k + "_upper", max, "double", "time", "both", 60,
self.dmax, group, self.ganglia_spoof_host)
g.send(k + "_count", count, "double", "count", "both", 60,
self.dmax, group, self.ganglia_spoof_host)
g.send(k + "_" + str(self.pct_threshold) + "pct",
max_threshold, "double", "time", "both", 60,
self.dmax, group, self.ganglia_spoof_host)
stats += 1
if self.transport == 'graphite':
stat_string += "statsd.numStats %s %d\n" % (stats, ts)
graphite = socket()
graphite.connect((self.graphite_host, self.graphite_port))
graphite.sendall(stat_string)
graphite.close()
self._set_timer()
if self.debug:
print "\n================== Flush completed. Waiting until next flush. Sent out %d metrics =======" % (
stats)
def flush(self):
ts = int(time.time())
stats = 0
if self.transport == 'graphite':
stat_string = ''
elif self.transport == 'ganglia':
g = gmetric.Gmetric(self.ganglia_host, self.ganglia_port,
self.ganglia_protocol)
for k, (v, t) in self.counters.items():
if self.expire > 0 and t + self.expire < ts:
if self.debug:
print("Expiring counter %s (age: %s)" % (k, ts - t))
del (self.counters[k])
continue
v = float(v)
v = v if self.no_aggregate_counters else v / (self.flush_interval /
1000)
if self.debug:
print("Sending %s => count=%s" % (k, v))
if self.transport == 'graphite':
msg = '%s.%s %s %s\n' % (self.counters_prefix, k, v, ts)
stat_string += msg
elif self.transport == 'ganglia':
# We put counters in _counters group. Underscore is to make sure counters show up
# first in the GUI. Change below if you disagree
if len(k) >= self.ganglia_max_length:
log.debug("Ganglia metric too long. Ignoring: %s" % k)
else:
g.send(k, v, "double", "count", "both", 60, self.dmax,
"_counters", self.ganglia_spoof_host)
elif self.transport == 'ganglia-gmetric':
self.send_to_ganglia_using_gmetric(k, v, "_counters", "count")
# Clear the counter once the data is sent
del (self.counters[k])
stats += 1
for k, (v, t) in self.gauges.items():
if self.expire > 0 and t + self.expire < ts:
if self.debug:
print("Expiring gauge %s (age: %s)" % (k, ts - t))
del (self.gauges[k])
continue
v = float(v)
if self.debug:
print("Sending %s => value=%s" % (k, v))
if self.transport == 'graphite':
# note: counters and gauges implicitly end up in the same namespace
msg = '%s.%s %s %s\n' % (self.counters_prefix, k, v, ts)
stat_string += msg
elif self.transport == 'ganglia':
if len(k) >= self.ganglia_max_length:
log.debug("Ganglia metric too long. Ignoring: %s" % k)
else:
g.send(k, v, "double", "count", "both", 60, self.dmax,
"_gauges", self.ganglia_spoof_host)
elif self.transport == 'ganglia-gmetric':
self.send_to_ganglia_using_gmetric(k, v, "_gauges", "gauge")
stats += 1
for k, (v, t) in self.timers.items():
if self.expire > 0 and t + self.expire < ts:
if self.debug:
print("Expiring timer %s (age: %s)" % (k, ts - t))
del (self.timers[k])
continue
if len(v) > 0:
# Sort all the received values. We need it to extract percentiles
v.sort()
count = len(v)
min = v[0]
max = v[-1]
mean = min
max_threshold = max
if count > 1:
thresh_index = int((self.pct_threshold / 100.0) * count)
max_threshold = v[thresh_index - 1]
total = sum(v)
mean = total / count
del (self.timers[k])
if self.debug:
print("Sending %s ====> lower=%s, mean=%s, upper=%s, %dpct=%s, count=%s" \
% (k, min, mean, max, self.pct_threshold, max_threshold, count))
if self.transport == 'graphite':
stat_string += TIMER_MSG % {
'prefix': self.timers_prefix,
'key': k,
'mean': mean,
'max': max,
'min': min,
'count': count,
'max_threshold': max_threshold,
'pct_threshold': self.pct_threshold,
'ts': ts,
}
elif self.transport == 'ganglia':
# We are gonna convert all times into seconds, then let rrdtool add proper SI unit. This avoids things like
# 3521 k ms which is 3.521 seconds
# What group should these metrics be in. For the time being we'll set it to the name of the key
if len(k) >= self.ganglia_max_length:
log.debug("Ganglia metric too long. Ignoring: %s" % k)
else:
group = k
g.send(k + "_min", min / 1000, "double", "seconds",
"both", 60, self.dmax, group,
self.ganglia_spoof_host)
g.send(k + "_mean", mean / 1000, "double", "seconds",
"both", 60, self.dmax, group,
self.ganglia_spoof_host)
g.send(k + "_max", max / 1000, "double", "seconds",
"both", 60, self.dmax, group,
self.ganglia_spoof_host)
g.send(k + "_count", count, "double", "count", "both",
60, self.dmax, group, self.ganglia_spoof_host)
g.send(k + "_" + str(self.pct_threshold) + "pct",
max_threshold / 1000, "double", "seconds",
"both", 60, self.dmax, group,
self.ganglia_spoof_host)
elif self.transport == 'ganglia-gmetric':
# We are gonna convert all times into seconds, then let rrdtool add proper SI unit. This avoids things like
# 3521 k ms which is 3.521 seconds
group = k
self.send_to_ganglia_using_gmetric(k + "_mean",
mean / 1000, group,
"seconds")
self.send_to_ganglia_using_gmetric(k + "_min", min / 1000,
group, "seconds")
self.send_to_ganglia_using_gmetric(k + "_max", max / 1000,
group, "seconds")
self.send_to_ganglia_using_gmetric(k + "_count", count,
group, "count")
self.send_to_ganglia_using_gmetric(
k + "_" + str(self.pct_threshold) + "pct",
max_threshold / 1000, group, "seconds")
stats += 1
if self.transport == 'graphite':
stat_string += "statsd.numStats %s %d\n" % (stats, ts)
# Prepend stats with Hosted Graphite API key if necessary
if self.global_prefix:
stat_string = '\n'.join([
'%s.%s' % (self.global_prefix, s)
for s in stat_string.split('\n')[:-1]
])
graphite = socket.socket()
try:
graphite.connect((self.graphite_host, self.graphite_port))
graphite.sendall(bytes(bytearray(stat_string, "utf-8")))
graphite.close()
except socket.error as e:
log.error("Error communicating with Graphite: %s" % e)
if self.debug:
print("Error communicating with Graphite: %s" % e)
if self.debug:
print("\n================== Flush completed. Waiting until next flush. Sent out %d metrics =======" \
% (stats))
def start_flush(self):
self.g = gmetric.Gmetric(self.host, self.port, self.protocol)