def grok_graphite_data(self, data):
STATS.incr('ts.graphite.grok.data', 1)
forwards = {}
for line in data.splitlines():
elts = line.split(' ')
elts = [s.strip() for s in elts if s.strip()]
if len(elts) != 3:
return
mname, value, timestamp = elts[0], elts[1], elts[2]
hkey = hashlib.sha1(mname).hexdigest()
ts_node_manager = self.clust.find_ts_node(hkey)
# if it's me that manage this key, I add it in my backend
if ts_node_manager == self.clust.uuid:
logger.debug("I am the TS node manager")
try:
timestamp = int(timestamp)
except ValueError:
return
value = to_best_int_float(value)
if value is None:
continue
self.tsb.add_value(timestamp, mname, value)
# not me? stack a forwarder
else:
logger.debug("The node manager for this Ts is ", ts_node_manager)
l = forwards.get(ts_node_manager, [])
l.append(line)
forwards[ts_node_manager] = l
for (uuid, lst) in forwards.iteritems():
node = self.clust.nodes.get(uuid, None)
# maybe the node disapear? bail out, we are not lucky
if node is None:
continue
packets = []
# first compute the packets
buf = ''
for line in lst:
buf += line+'\n'
if len(buf) > 1024:
packets.append(buf)
buf = ''
if buf != '':
packets.append(buf)
# UDP
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
for packet in packets:
# do NOT use the node['port'], it's the internal communication, not the graphite one!
sock.sendto(packet, (node['addr'], self.graphite_port))
sock.close()
'''
def launch_statsd_udp_listener(self):
self.udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
self.udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1048576)
self.udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.log(self.udp_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
self.udp_sock.bind((self.addr, self.statsd_port))
self.log("TS UDP port open", self.statsd_port)
self.log("UDP RCVBUF", self.udp_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
while True:#not self.interrupted:
try:
data, addr = self.udp_sock.recvfrom(65535) # buffer size is 1024 bytes
except socket.timeout: # loop until we got something
continue
self.log("UDP: received message:", data, addr)
# No data? bail out :)
if len(data) == 0:
continue
self.log("GETDATA", data)
for line in data.splitlines():
# avoid invalid lines
if not '|' in line:
continue
elts = line.split('|', 1)
# invalid, no type in the right part
if len(elts) == 1:
continue
_name_value = elts[0].strip()
# maybe it's an invalid name...
if not ':' in _name_value:
continue
_nvs = _name_value.split(':')
if len(_nvs) != 2:
continue
mname = _nvs[0].strip()
# Two cases: it's for me or not
hkey = hashlib.sha1(mname).hexdigest()
ts_node_manager = self.clust.find_ts_node(hkey)
# if it's me that manage this key, I add it in my backend
if ts_node_manager != self.clust.uuid:
node = self.clust.nodes.get(ts_node_manager, None)
# threads are dangerous things...
if node is None:
continue
# TODO: do bulk send of this, like for graphite
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# do NOT use the node['port'], it's the internal communication, not the graphite one!
sock.sendto(line, (node['addr'], self.statsd_port))
sock.close()
continue
# Here we are sure it's really for us, so manage it :)
value = to_best_int_float(_nvs[1].strip())
if not mname or value is None:
continue
# Look at the type of the data
_type = elts[1].strip()
if len(_type) == 0:
continue
## Gauge: <metric name>:<value>|g
elif _type == 'g':
self.nb_data += 1
self.log('GAUGE', mname, value)
with self.stats_lock:
gentry = self.gauges.get(mname, None)
if gentry is None:
# sum, nb, min, max
gentry = (0.0, 0, None, None)
_sum, nb, _min, _max = gentry
_sum += value
nb += 1
if _min is None or value < _min:
_min = value
if _max is None or value > _max:
_max = value
self.gauges[mname] = (_sum, nb, _min, _max)
self.log('NEW GAUGE', mname, self.gauges[mname])
## Timers: <metric name>:<value>|ms
## But also
## Histograms: <metric name>:<value>|h
elif _type == 'ms' or _type == 'h':
self.log('timers', mname, value)
# TODO: avoit the SET each time
timer = self.timers.get(mname, [])
timer.append(value)
self.timers[mname] = timer
## Counters: <metric name>:<value>|c[|@<sample rate>]
elif _type == 'c':
self.nb_data += 1
self.log('COUNTER', mname, value, "rate", 1)
with self.stats_lock:
cvalue, ccount = self.counters.get(mname, (0,0))
self.counters[mname] = (cvalue + value, ccount + 1)
self.log('NEW COUNTER', mname, self.counters[mname])
## Meters: <metric name>:<value>|m
elif _type == 'm':
self.log('METERs', mname, value)
else: # unknow type, maybe a c[|@<sample rate>]
if _type[0] == 'c':
self.nb_data += 1
if not '|' in _type:
continue
srate = _type.split('|')[1].strip()
if len(srate) == 0 or srate[0] != '@':
continue
try:
rate = float(srate[1:])
except ValueError:
continue
# Invalid rate, 0.0 is invalid too ;)
if rate <= 0.0 or rate > 1.0:
continue
self.log('COUNTER', mname, value, "rate", rate)
with self.stats_lock:
cvalue, ccount = self.counters.get(mname, (0,0))
self.log('INCR counter', (value/rate))
self.counters[mname] = (cvalue + (value/rate), ccount + 1/rate)
self.log('NEW COUNTER', mname, self.counters[mname])
if True: # or ('MongoDBDBStats' in self.config and self.config['MongoDBDBStats'] == 'yes':
logger.debug('getMongoDBStatus: db.stats() too')
status['dbStats'] = {}
for database in conn.database_names():
if database != 'config' and database != 'local' and database != 'admin' and database != 'test':
logger.debug('getMongoDBStatus: executing db.stats() for %s', database)
status['dbStats'][database] = conn[database].command('dbstats')
status['dbStats'][database]['namespaces'] = conn[database]['system']['namespaces'].count()
# Ensure all strings to prevent JSON parse errors. We typecast on the server
for key in status['dbStats'][database].keys():
status['dbStats'][database][key] = str(status['dbStats'][database][key])
# try a float/int cast
v = to_best_int_float(status['dbStats'][database][key])
if v is not None:
status['dbStats'][database][key] = v
except Exception, ex:
logger.error('Unable to get MongoDB status - Exception = %s', traceback.format_exc())
return False
logger.debug('getMongoDBStatus: completed, returning')
return status
def setMongoDBStore(self, statusOutput):
