def grok_graphite_data(self, data): STATS.incr('ts.graphite.grok.data', 1) forwards = {} for line in data.splitlines(): elts = line.split(' ') elts = [s.strip() for s in elts if s.strip()] if len(elts) != 3: return mname, value, timestamp = elts[0], elts[1], elts[2] hkey = hashlib.sha1(mname).hexdigest() ts_node_manager = self.clust.find_ts_node(hkey) # if it's me that manage this key, I add it in my backend if ts_node_manager == self.clust.uuid: logger.debug("I am the TS node manager") try: timestamp = int(timestamp) except ValueError: return value = to_best_int_float(value) if value is None: continue self.tsb.add_value(timestamp, mname, value) # not me? stack a forwarder else: logger.debug("The node manager for this Ts is ", ts_node_manager) l = forwards.get(ts_node_manager, []) l.append(line) forwards[ts_node_manager] = l for (uuid, lst) in forwards.iteritems(): node = self.clust.nodes.get(uuid, None) # maybe the node disapear? bail out, we are not lucky if node is None: continue packets = [] # first compute the packets buf = '' for line in lst: buf += line+'\n' if len(buf) > 1024: packets.append(buf) buf = '' if buf != '': packets.append(buf) # UDP sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) for packet in packets: # do NOT use the node['port'], it's the internal communication, not the graphite one! sock.sendto(packet, (node['addr'], self.graphite_port)) sock.close() '''
def launch_statsd_udp_listener(self): self.udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP self.udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1048576) self.udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.log(self.udp_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)) self.udp_sock.bind((self.addr, self.statsd_port)) self.log("TS UDP port open", self.statsd_port) self.log("UDP RCVBUF", self.udp_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)) while True:#not self.interrupted: try: data, addr = self.udp_sock.recvfrom(65535) # buffer size is 1024 bytes except socket.timeout: # loop until we got something continue self.log("UDP: received message:", data, addr) # No data? bail out :) if len(data) == 0: continue self.log("GETDATA", data) for line in data.splitlines(): # avoid invalid lines if not '|' in line: continue elts = line.split('|', 1) # invalid, no type in the right part if len(elts) == 1: continue _name_value = elts[0].strip() # maybe it's an invalid name... if not ':' in _name_value: continue _nvs = _name_value.split(':') if len(_nvs) != 2: continue mname = _nvs[0].strip() # Two cases: it's for me or not hkey = hashlib.sha1(mname).hexdigest() ts_node_manager = self.clust.find_ts_node(hkey) # if it's me that manage this key, I add it in my backend if ts_node_manager != self.clust.uuid: node = self.clust.nodes.get(ts_node_manager, None) # threads are dangerous things... if node is None: continue # TODO: do bulk send of this, like for graphite sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # do NOT use the node['port'], it's the internal communication, not the graphite one! sock.sendto(line, (node['addr'], self.statsd_port)) sock.close() continue # Here we are sure it's really for us, so manage it :) value = to_best_int_float(_nvs[1].strip()) if not mname or value is None: continue # Look at the type of the data _type = elts[1].strip() if len(_type) == 0: continue ## Gauge: <metric name>:<value>|g elif _type == 'g': self.nb_data += 1 self.log('GAUGE', mname, value) with self.stats_lock: gentry = self.gauges.get(mname, None) if gentry is None: # sum, nb, min, max gentry = (0.0, 0, None, None) _sum, nb, _min, _max = gentry _sum += value nb += 1 if _min is None or value < _min: _min = value if _max is None or value > _max: _max = value self.gauges[mname] = (_sum, nb, _min, _max) self.log('NEW GAUGE', mname, self.gauges[mname]) ## Timers: <metric name>:<value>|ms ## But also ## Histograms: <metric name>:<value>|h elif _type == 'ms' or _type == 'h': self.log('timers', mname, value) # TODO: avoit the SET each time timer = self.timers.get(mname, []) timer.append(value) self.timers[mname] = timer ## Counters: <metric name>:<value>|c[|@<sample rate>] elif _type == 'c': self.nb_data += 1 self.log('COUNTER', mname, value, "rate", 1) with self.stats_lock: cvalue, ccount = self.counters.get(mname, (0,0)) self.counters[mname] = (cvalue + value, ccount + 1) self.log('NEW COUNTER', mname, self.counters[mname]) ## Meters: <metric name>:<value>|m elif _type == 'm': self.log('METERs', mname, value) else: # unknow type, maybe a c[|@<sample rate>] if _type[0] == 'c': self.nb_data += 1 if not '|' in _type: continue srate = _type.split('|')[1].strip() if len(srate) == 0 or srate[0] != '@': continue try: rate = float(srate[1:]) except ValueError: continue # Invalid rate, 0.0 is invalid too ;) if rate <= 0.0 or rate > 1.0: continue self.log('COUNTER', mname, value, "rate", rate) with self.stats_lock: cvalue, ccount = self.counters.get(mname, (0,0)) self.log('INCR counter', (value/rate)) self.counters[mname] = (cvalue + (value/rate), ccount + 1/rate) self.log('NEW COUNTER', mname, self.counters[mname])
if True: # or ('MongoDBDBStats' in self.config and self.config['MongoDBDBStats'] == 'yes': logger.debug('getMongoDBStatus: db.stats() too') status['dbStats'] = {} for database in conn.database_names(): if database != 'config' and database != 'local' and database != 'admin' and database != 'test': logger.debug('getMongoDBStatus: executing db.stats() for %s', database) status['dbStats'][database] = conn[database].command('dbstats') status['dbStats'][database]['namespaces'] = conn[database]['system']['namespaces'].count() # Ensure all strings to prevent JSON parse errors. We typecast on the server for key in status['dbStats'][database].keys(): status['dbStats'][database][key] = str(status['dbStats'][database][key]) # try a float/int cast v = to_best_int_float(status['dbStats'][database][key]) if v is not None: status['dbStats'][database][key] = v except Exception, ex: logger.error('Unable to get MongoDB status - Exception = %s', traceback.format_exc()) return False logger.debug('getMongoDBStatus: completed, returning') return status def setMongoDBStore(self, statusOutput):