def create_connection(self): self.con = HTTPClient(address=self.arb_satmap['address'], port=self.arb_satmap['port'], timeout=self.timeout, data_timeout=self.data_timeout, use_ssl=self.use_ssl) self.uri = self.con.uri
def __init__(self): self.name = '' self.type = '' self.app = None self.stats = {} self.api_key = '' self.secret = '' self.con = HTTPClient(uri='http://kernel.shinken.io')
class Stats(object): def __init__(self): self.name = '' self.type = '' self.app = None self.stats = {} # There are two modes that are not exclusive # first the kernel mode self.api_key = '' self.secret = '' self.http_proxy = '' self.con = HTTPClient(uri='http://kernel.shinken.io') # then the statsd one self.statsd_sock = None self.statsd_addr = None def launch_reaper_thread(self): self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper') self.reaper_thread.daemon = True self.reaper_thread.start() def register(self, app, name, _type, api_key='', secret='', http_proxy='', statsd_host='localhost', statsd_port=8125, statsd_prefix='shinken', statsd_enabled=False): self.app = app self.name = name self.type = _type # kernel.io part self.api_key = api_key self.secret = secret self.http_proxy = http_proxy # local statsd part self.statsd_host = statsd_host self.statsd_port = statsd_port self.statsd_prefix = statsd_prefix self.statsd_enabled = statsd_enabled if self.statsd_enabled: logger.debug('Loading statsd communication with %s:%s.%s', self.statsd_host, self.statsd_port, self.statsd_prefix) self.load_statsd() # Also load the proxy if need self.con.set_proxy(self.http_proxy) # Let be crystal clear about why I don't use the statsd lib in python: it's crappy. # how guys did you f**k this up to this point? django by default for the conf?? really?... # So raw socket are far better here def load_statsd(self): try: self.statsd_addr = (socket.gethostbyname(self.statsd_host), self.statsd_port) self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) except (socket.error, socket.gaierror), exp: logger.error('Cannot create statsd socket: %s' % exp) return
def __init__(self): self.name = '' self.type = '' self.app = None self.stats = {} self.api_key = '' self.secret = '' self.cyph = None self.con = HTTPClient(uri='http://metrology')
def __init__(self): self.name = '' self.type = '' self.app = None self.stats = {} # There are two modes that are not exclusive # first the kernel mode self.api_key = '' self.secret = '' self.http_proxy = '' self.con = HTTPClient(uri='http://kernel.shinken.io') # then the statsd one self.statsd_sock = None self.statsd_addr = None
def pynag_con_init(self, id, type='scheduler'): # Get the good links tab for looping.. links = self.get_links_from_type(type) if links is None: logger.debug('Type unknown for connection! %s' % type) return if type == 'scheduler': # If sched is not active, I do not try to init # it is just useless is_active = links[id]['active'] if not is_active: return # If we try to connect too much, we slow down our tests if self.is_connection_try_too_close(links[id]): return # Ok, we can now update it links[id]['last_connection'] = time.time() # DBG: print "Init connection with", links[id]['uri'] running_id = links[id]['running_id'] # DBG: print "Running id before connection", running_id uri = links[id]['uri'] try: con = links[id]['con'] = HTTPClient( uri=uri, strong_ssl=links[id]['hard_ssl_name_check']) except HTTPExceptions, exp: # But the multiprocessing module is not compatible with it! # so we must disable it immediately after logger.info("Connection problem to the %s %s: %s" % (type, links[id]['name'], str(exp))) links[id]['con'] = None return
def create_connection(self): self.con = HTTPClient(address=self.arb_satmap['address'], port=self.arb_satmap['port'], timeout=self.timeout, data_timeout=self.data_timeout, use_ssl=self.use_ssl, strong_ssl=self.hard_ssl_name_check ) self.uri = self.con.uri
def __init__(self): self.name = '' self.type = '' self.app = None self.stats = {} # There are two modes that are not exclusive # first the kernel mode self.api_key = '' self.secret = '' self.http_proxy = '' self.con = HTTPClient(uri='http://kernel.shinken.io') # then the statsd one self.statsd_interval = 5 self.statsd_types = ['system', 'queue', 'perf'] self.statsd_sock = None self.statsd_addr = None self.statsd_types = 'system,object,queue,perf' self.statsd_pattern = None self.name_cache = {}
def __init__(self): self.name = "" self.type = "" self.app = None self.stats = {} # There are two modes that are not exclusive # first the kernel mode self.api_key = "" self.secret = "" self.http_proxy = "" self.con = HTTPClient(uri="http://kernel.shinken.io") # then the statsd one self.statsd_interval = 5 self.statsd_types = ["system", "queue", "perf"] self.statsd_sock = None self.statsd_addr = None self.statsd_types = "system,object,queue,perf" self.statsd_pattern = None self.name_cache = {}
class Stats(object): def __init__(self): self.name = '' self.type = '' self.app = None self.stats = {} self.api_key = '' self.secret = '' self.cyph = None self.con = HTTPClient(uri='http://metrology') def launch_reaper_thread(self): self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper') self.reaper_thread.daemon = True self.reaper_thread.start() def register(self, app, name, _type, api_key='', secret=''): self.app = app self.name = name self.type = _type self.api_key = api_key self.secret = secret # Assumea 16 len secret, but should be alreayd ok self.secret += '\0' * (-len(self.secret) % 16) if AES is not None and self.secret != '': self.cyph = AES.new(self.secret, AES.MODE_ECB) # Will increment a stat key, if None, start at 0 def incr(self, k, v): _min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0)) nb += 1 _sum += v if _min is None or v < _min: _min = v if _max is None or v > _max: _max = v self.stats[k] = (_min, _max, nb, _sum) def reaper(self): while True: now = int(time.time()) logger.debug('REAPER loop') stats = self.stats self.stats = {} if len(stats) != 0: s = ', '.join( ['%s:%s' % (k, v) for (k, v) in stats.iteritems()]) logger.debug("REAPER: %s ", s) # If we are not in an initializer daemon we skip, we cannot have a real name, it sucks # to find the data after this if not self.name: time.sleep(10) continue logger.debug('REAPER we got a name') metrics = [] for (k, e) in stats.iteritems(): nk = '%s.%s.%s' % (self.type, self.name, k) logger.debug('REAP %s:%s', nk, e) _min, _max, nb, _sum = e _avg = float(_sum) / nb # nb can't be 0 here and _min_max can't be None too s = '%s.avg %f %d' % (nk, _avg, now) metrics.append(s) s = '%s.min %f %d' % (nk, _min, now) metrics.append(s) s = '%s.max %f %d' % (nk, _max, now) metrics.append(s) s = '%s.count %f %d' % (nk, nb, now) metrics.append(s) logger.debug('REAPER metrics to send %s (%d)', metrics, len(str(metrics))) # get the inner data for the daemon struct = self.app.get_stats_struct() struct['metrics'].extend(metrics) logger.debug('REAPER whole struct %s', struct) j = json.dumps(struct) if self.cyph is not None: logger.debug('PUT to /api/v1/put/ with %s %s', self.api_key, self.secret) # assume a %16 length messagexs j += '\0' * (-len(j) % 16) encrypted_text = self.cyph.encrypt(j) try: self.con.put('/api/v1/put/', encrypted_text) except HTTPException, exp: logger.debug('REAPER cannot put to the metric server %s', exp) time.sleep(10)
class Stats(object): def __init__(self): self.name = '' self.type = '' self.app = None self.stats = {} # There are two modes that are not exclusive # first the kernel mode self.api_key = '' self.secret = '' self.http_proxy = '' self.con = HTTPClient(uri='http://kernel.shinken.io') # then the statsd one self.statsd_interval = 5 self.statsd_types = ['system', 'queue', 'perf'] self.statsd_sock = None self.statsd_addr = None self.statsd_types = 'system,object,queue,perf' self.statsd_pattern = None self.name_cache = {} def launch_reaper_thread(self): self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper') self.reaper_thread.daemon = True self.reaper_thread.start() def launch_harvester_thread(self): self.harvester_thread = threading.Thread(None, target=self.harvester, name='stats-harvester') self.harvester_thread.daemon = True self.harvester_thread.start() def register(self, app, name, _type, api_key='', secret='', http_proxy='', statsd_host='localhost', statsd_port=8125, statsd_prefix='shinken', statsd_enabled=False, statsd_interval=5, statsd_types=None, statsd_pattern=None): self.app = app self.name = name self.type = _type # kernel.io part self.api_key = api_key self.secret = secret self.http_proxy = http_proxy # local statsd part self.statsd_host = statsd_host self.statsd_port = statsd_port self.statsd_prefix = statsd_prefix self.statsd_enabled = statsd_enabled self.statsd_interval = statsd_interval if statsd_types is not None: self.statsd_types = [ t.strip() for t in statsd_types.split(",") if t.strip() ] if statsd_pattern is not None: self.statsd_pattern = statsd_pattern self.name_cache = {} if self.statsd_enabled: logger.debug('Loading statsd communication with %s:%s.%s', self.statsd_host, self.statsd_port, self.statsd_prefix) self.load_statsd() # Also load the proxy if need self.con.set_proxy(self.http_proxy) # Tells whether statsd is enabled and stats should be sent def is_statsd_enabled(self): return (self.statsd_sock is not None and self.name and self.app is not None) # Tells whether kernel.status.io exporter is enabled def is_shinkenio_enabled(self): return (self.name and self.api_key and self.secret) # Let be crystal clear about why I don't use the statsd lib in python: it's crappy. # how guys did you f**k this up to this point? django by default for the conf?? really?... # So raw socket are far better here def load_statsd(self): try: self.statsd_addr = (socket.gethostbyname(self.statsd_host), self.statsd_port) self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) except (socket.error, socket.gaierror), exp: logger.error('Cannot create statsd socket: %s' % exp) return
class SatelliteLink(Item): """SatelliteLink is a common Class for link to satellite for Arbiter with Conf Dispatcher. """ # id = 0 each Class will have it's own id properties = Item.properties.copy() properties.update({ 'address': StringProp(fill_brok=['full_status']), 'timeout': IntegerProp(default='3', fill_brok=['full_status']), 'data_timeout': IntegerProp(default='120', fill_brok=['full_status']), 'check_interval': IntegerProp(default='60', fill_brok=['full_status']), 'max_check_attempts': IntegerProp(default='3', fill_brok=['full_status']), 'spare': BoolProp(default='0', fill_brok=['full_status']), 'manage_sub_realms': BoolProp(default='1', fill_brok=['full_status']), 'manage_arbiters': BoolProp(default='0', fill_brok=['full_status'], to_send=True), 'modules': ListProp(default='', to_send=True), 'polling_interval': IntegerProp(default='1', fill_brok=['full_status'], to_send=True), 'use_timezone': StringProp(default='NOTSET', to_send=True), 'realm': StringProp(default='', fill_brok=['full_status'], brok_transformation=get_obj_name_two_args_and_void), 'satellitemap': DictProp(default=None, elts_prop=AddrProp, to_send=True, override=True), 'use_ssl': BoolProp(default='0', fill_brok=['full_status']), }) running_properties = Item.running_properties.copy() running_properties.update({ 'con': StringProp(default=None), 'alive': StringProp(default=True, fill_brok=['full_status']), 'broks': StringProp(default=[]), 'attempt': StringProp(default=0, fill_brok=['full_status']), # the number of failed attempt 'reachable': StringProp(default=False, fill_brok=[ 'full_status' ]), # can be network ask or not (dead or check in timeout or error) 'last_check': IntegerProp(default=0, fill_brok=['full_status']), 'managed_confs': StringProp(default={}), }) def __init__(self, *args, **kwargs): super(SatelliteLink, self).__init__(*args, **kwargs) self.arb_satmap = {'address': '0.0.0.0', 'port': 0} if hasattr(self, 'address'): self.arb_satmap['address'] = self.address if hasattr(self, 'port'): try: self.arb_satmap['port'] = int(self.port) except: pass def set_arbiter_satellitemap(self, satellitemap): """ arb_satmap is the satellitemap in current context: - A SatelliteLink is owned by an Arbiter - satellitemap attribute of SatelliteLink is the map defined IN THE satellite configuration but for creating connections, we need the have the satellitemap of the Arbiter """ self.arb_satmap = {'address': self.address, 'port': self.port} self.arb_satmap.update(satellitemap) def create_connection(self): self.con = HTTPClient(address=self.arb_satmap['address'], port=self.arb_satmap['port'], timeout=self.timeout, data_timeout=self.data_timeout, use_ssl=self.use_ssl) self.uri = self.con.uri def put_conf(self, conf): if self.con is None: self.create_connection() # Maybe the connexion was not ok, bail out if not self.con: return False try: #pyro.set_timeout(self.con, self.data_timeout) self.con.post('put_conf', {'conf': conf}, wait='long') #pyro.set_timeout(self.con, self.timeout) print "PUT CONF SUCESS", self.get_name() return True except HTTPExceptions, exp: self.con = None logger.error("Failed sending configuration for %s: %s" % (self.get_name(), str(exp))) return False
class Stats(object): def __init__(self): self.name = '' self.type = '' self.app = None self.stats = {} self.api_key = '' self.secret = '' self.cyph = None self.con = HTTPClient(uri='http://metrology') def launch_reaper_thread(self): self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper') self.reaper_thread.daemon = True self.reaper_thread.start() def register(self, app, name, _type, api_key='', secret=''): self.app = app self.name = name self.type = _type self.api_key = api_key self.secret = secret # Assumea 16 len secret, but should be alreayd ok self.secret += '\0' * (-len(self.secret) % 16) if AES is not None and self.secret != '': self.cyph = AES.new(self.secret, AES.MODE_ECB) # Will increment a stat key, if None, start at 0 def incr(self, k, v): _min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0)) nb += 1 _sum += v if _min is None or v < _min: _min = v if _max is None or v > _max: _max = v self.stats[k] = (_min, _max, nb, _sum) def reaper(self): while True: now = int(time.time()) logger.debug('REAPER loop') stats = self.stats self.stats = {} if len(stats) != 0: s = ', '.join(['%s:%s' % (k,v) for (k,v) in stats.iteritems()]) logger.debug("REAPER: %s ", s) # If we are not in an initializer daemon we skip, we cannot have a real name, it sucks # to find the data after this if not self.name: time.sleep(10) continue logger.debug('REAPER we got a name') metrics = [] for (k,e) in stats.iteritems(): nk = '%s.%s.%s' % (self.type, self.name, k) logger.debug('REAP %s:%s', nk, e) _min, _max, nb, _sum = e _avg = float(_sum) / nb # nb can't be 0 here and _min_max can't be None too s = '%s.avg %f %d' % (nk, _avg, now) metrics.append(s) s = '%s.min %f %d' % (nk, _min, now) metrics.append(s) s = '%s.max %f %d' % (nk, _max, now) metrics.append(s) s = '%s.count %f %d' % (nk, nb, now) metrics.append(s) logger.debug('REAPER metrics to send %s (%d)', metrics, len(str(metrics)) ) # get the inner data for the daemon struct = self.app.get_stats_struct() struct['metrics'].extend(metrics) logger.debug('REAPER whole struct %s', struct) j = json.dumps(struct) if self.cyph is not None: logger.debug('PUT to /api/v1/put/ with %s %s', self.api_key, self.secret) # assume a %16 length messagexs j += '\0' * (-len(j) % 16) encrypted_text = self.cyph.encrypt(j) try: self.con.put('/api/v1/put/', encrypted_text) except HTTPException, exp: logger.debug('REAPER cannot put to the metric server %s', exp) time.sleep(10)
class Stats(object): def __init__(self): self.name = "" self.type = "" self.app = None self.stats = {} # There are two modes that are not exclusive # first the kernel mode self.api_key = "" self.secret = "" self.http_proxy = "" self.con = HTTPClient(uri="http://kernel.shinken.io") # then the statsd one self.statsd_interval = 5 self.statsd_types = ["system", "queue", "perf"] self.statsd_sock = None self.statsd_addr = None self.statsd_types = "system,object,queue,perf" self.statsd_pattern = None self.name_cache = {} def launch_reaper_thread(self): self.reaper_thread = threading.Thread(None, target=self.reaper, name="stats-reaper") self.reaper_thread.daemon = True self.reaper_thread.start() def launch_harvester_thread(self): self.harvester_thread = threading.Thread(None, target=self.harvester, name="stats-harvester") self.harvester_thread.daemon = True self.harvester_thread.start() def register( self, app, name, _type, api_key="", secret="", http_proxy="", statsd_host="localhost", statsd_port=8125, statsd_prefix="shinken", statsd_enabled=False, statsd_interval=5, statsd_types=None, statsd_pattern=None, ): self.app = app self.name = name self.type = _type # kernel.io part self.api_key = api_key self.secret = secret self.http_proxy = http_proxy # local statsd part self.statsd_host = statsd_host self.statsd_port = statsd_port self.statsd_prefix = statsd_prefix self.statsd_enabled = statsd_enabled self.statsd_interval = statsd_interval if statsd_types is not None: self.statsd_types = [t.strip() for t in statsd_types.split(",") if t.strip()] if statsd_pattern is not None: self.statsd_pattern = statsd_pattern self.name_cache = {} if self.statsd_enabled: logger.debug( "Loading statsd communication with %s:%s.%s", self.statsd_host, self.statsd_port, self.statsd_prefix ) self.load_statsd() # Also load the proxy if need self.con.set_proxy(self.http_proxy) # Tells whether statsd is enabled and stats should be sent def is_statsd_enabled(self): return self.statsd_sock is not None and self.name and self.app is not None # Tells whether kernel.status.io exporter is enabled def is_shinkenio_enabled(self): return self.name and self.api_key and self.secret # Let be crystal clear about why I don't use the statsd lib in python: it's crappy. # how guys did you f**k this up to this point? django by default for the conf?? really?... # So raw socket are far better here def load_statsd(self): try: self.statsd_addr = (socket.gethostbyname(self.statsd_host), self.statsd_port) self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) except (socket.error, socket.gaierror), exp: logger.error("Cannot create statsd socket: %s" % exp) return
class SatelliteLink(Item): """SatelliteLink is a common Class for link to satellite for Arbiter with Conf Dispatcher. """ # id = 0 each Class will have it's own id properties = Item.properties.copy() properties.update( { "address": StringProp(fill_brok=["full_status"]), "timeout": IntegerProp(default="3", fill_brok=["full_status"]), "data_timeout": IntegerProp(default="120", fill_brok=["full_status"]), "check_interval": IntegerProp(default="60", fill_brok=["full_status"]), "max_check_attempts": IntegerProp(default="3", fill_brok=["full_status"]), "spare": BoolProp(default="0", fill_brok=["full_status"]), "manage_sub_realms": BoolProp(default="1", fill_brok=["full_status"]), "manage_arbiters": BoolProp(default="0", fill_brok=["full_status"], to_send=True), "modules": ListProp(default="", to_send=True), "polling_interval": IntegerProp(default="1", fill_brok=["full_status"], to_send=True), "use_timezone": StringProp(default="NOTSET", to_send=True), "realm": StringProp( default="", fill_brok=["full_status"], brok_transformation=get_obj_name_two_args_and_void ), "satellitemap": DictProp(default=None, elts_prop=AddrProp, to_send=True, override=True), "use_ssl": BoolProp(default="0", fill_brok=["full_status"]), "hard_ssl_name_check": BoolProp(default="0", fill_brok=["full_status"]), "passive": BoolProp(default="0", fill_brok=["full_status"], to_send=True), } ) running_properties = Item.running_properties.copy() running_properties.update( { "con": StringProp(default=None), "alive": StringProp(default=True, fill_brok=["full_status"]), "broks": StringProp(default=[]), "attempt": StringProp(default=0, fill_brok=["full_status"]), # the number of failed attempt "reachable": StringProp( default=False, fill_brok=["full_status"] ), # can be network ask or not (dead or check in timeout or error) "last_check": IntegerProp(default=0, fill_brok=["full_status"]), "managed_confs": StringProp(default={}), } ) def __init__(self, *args, **kwargs): super(SatelliteLink, self).__init__(*args, **kwargs) self.arb_satmap = {"address": "0.0.0.0", "port": 0} if hasattr(self, "address"): self.arb_satmap["address"] = self.address if hasattr(self, "port"): try: self.arb_satmap["port"] = int(self.port) except: pass def set_arbiter_satellitemap(self, satellitemap): """ arb_satmap is the satellitemap in current context: - A SatelliteLink is owned by an Arbiter - satellitemap attribute of SatelliteLink is the map defined IN THE satellite configuration but for creating connections, we need the have the satellitemap of the Arbiter """ self.arb_satmap = { "address": self.address, "port": self.port, "use_ssl": self.use_ssl, "hard_ssl_name_check": self.hard_ssl_name_check, } self.arb_satmap.update(satellitemap) def create_connection(self): self.con = HTTPClient( address=self.arb_satmap["address"], port=self.arb_satmap["port"], timeout=self.timeout, data_timeout=self.data_timeout, use_ssl=self.use_ssl, strong_ssl=self.hard_ssl_name_check, ) self.uri = self.con.uri def put_conf(self, conf): if self.con is None: self.create_connection() # Maybe the connexion was not ok, bail out if not self.con: return False try: # pyro.set_timeout(self.con, self.data_timeout) self.con.get("ping") self.con.post("put_conf", {"conf": conf}, wait="long") # pyro.set_timeout(self.con, self.timeout) print "PUT CONF SUCESS", self.get_name() return True except HTTPExceptions, exp: self.con = None logger.error("Failed sending configuration for %s: %s" % (self.get_name(), str(exp))) return False
class Stats(object): def __init__(self): self.name = '' self.type = '' self.app = None self.stats = {} self.api_key = '' self.secret = '' self.con = HTTPClient(uri='http://kernel.shinken.io') def launch_reaper_thread(self): self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper') self.reaper_thread.daemon = True self.reaper_thread.start() def register(self, app, name, _type, api_key='', secret=''): self.app = app self.name = name self.type = _type self.api_key = api_key self.secret = secret # Will increment a stat key, if None, start at 0 def incr(self, k, v): _min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0)) nb += 1 _sum += v if _min is None or v < _min: _min = v if _max is None or v > _max: _max = v self.stats[k] = (_min, _max, nb, _sum) def _encrypt(self, data): m = hashlib.md5() m.update(self.secret) key = m.hexdigest() m = hashlib.md5() m.update(self.secret + key) iv = m.hexdigest() data = pad(data) aes = AES.new(key, AES.MODE_CBC, iv[:16]) encrypted = aes.encrypt(data) return base64.urlsafe_b64encode(encrypted) def reaper(self): while True: now = int(time.time()) stats = self.stats self.stats = {} if len(stats) != 0: s = ', '.join( ['%s:%s' % (k, v) for (k, v) in stats.iteritems()]) # If we are not in an initializer daemon we skip, we cannot have a real name, it sucks # to find the data after this if not self.name or not self.api_key or not self.secret: time.sleep(60) continue metrics = [] for (k, e) in stats.iteritems(): nk = '%s.%s.%s' % (self.type, self.name, k) _min, _max, nb, _sum = e _avg = float(_sum) / nb # nb can't be 0 here and _min_max can't be None too s = '%s.avg %f %d' % (nk, _avg, now) metrics.append(s) s = '%s.min %f %d' % (nk, _min, now) metrics.append(s) s = '%s.max %f %d' % (nk, _max, now) metrics.append(s) s = '%s.count %f %d' % (nk, nb, now) metrics.append(s) #logger.debug('REAPER metrics to send %s (%d)' % (metrics, len(str(metrics))) ) # get the inner data for the daemon struct = self.app.get_stats_struct() struct['metrics'].extend(metrics) #logger.debug('REAPER whole struct %s' % struct) j = json.dumps(struct) if AES is not None and self.secret != '': # RESET AFTER EACH calls! logger.debug( 'Stats PUT to kernel.shinken.io/api/v1/put/ with %s %s' % (self.api_key, self.secret)) # assume a %16 length messagexs encrypted_text = self._encrypt(j) try: r = self.con.put( '/api/v1/put/?api_key=%s' % (self.api_key), encrypted_text) except HTTPException, exp: logger.debug( 'Stats REAPER cannot put to the metric server %s' % exp) time.sleep(60)
class Stats(object): def __init__(self): self.name = '' self.type = '' self.app = None self.stats = {} self.api_key = '' self.secret = '' self.con = HTTPClient(uri='http://kernel.shinken.io') def launch_reaper_thread(self): self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper') self.reaper_thread.daemon = True self.reaper_thread.start() def register(self, app, name, _type, api_key='', secret=''): self.app = app self.name = name self.type = _type self.api_key = api_key self.secret = secret # Will increment a stat key, if None, start at 0 def incr(self, k, v): _min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0)) nb += 1 _sum += v if _min is None or v < _min: _min = v if _max is None or v > _max: _max = v self.stats[k] = (_min, _max, nb, _sum) def _encrypt(self, data): m = hashlib.md5() m.update(self.secret) key = m.hexdigest() m = hashlib.md5() m.update(self.secret + key) iv = m.hexdigest() data = pad(data) aes = AES.new(key, AES.MODE_CBC, iv[:16]) encrypted = aes.encrypt(data) return base64.urlsafe_b64encode(encrypted) def reaper(self): while True: now = int(time.time()) stats = self.stats self.stats = {} if len(stats) != 0: s = ', '.join(['%s:%s' % (k,v) for (k,v) in stats.iteritems()]) # If we are not in an initializer daemon we skip, we cannot have a real name, it sucks # to find the data after this if not self.name or not self.api_key or not self.secret: time.sleep(60) continue metrics = [] for (k,e) in stats.iteritems(): nk = '%s.%s.%s' % (self.type, self.name, k) _min, _max, nb, _sum = e _avg = float(_sum) / nb # nb can't be 0 here and _min_max can't be None too s = '%s.avg %f %d' % (nk, _avg, now) metrics.append(s) s = '%s.min %f %d' % (nk, _min, now) metrics.append(s) s = '%s.max %f %d' % (nk, _max, now) metrics.append(s) s = '%s.count %f %d' % (nk, nb, now) metrics.append(s) #logger.debug('REAPER metrics to send %s (%d)' % (metrics, len(str(metrics))) ) # get the inner data for the daemon struct = self.app.get_stats_struct() struct['metrics'].extend(metrics) #logger.debug('REAPER whole struct %s' % struct) j = json.dumps(struct) if AES is not None and self.secret != '': # RESET AFTER EACH calls! logger.debug('Stats PUT to kernel.shinken.io/api/v1/put/ with %s %s' % (self.api_key, self.secret)) # assume a %16 length messagexs encrypted_text = self._encrypt(j) try: r = self.con.put('/api/v1/put/?api_key=%s' % (self.api_key), encrypted_text) except HTTPException, exp: logger.debug('Stats REAPER cannot put to the metric server %s' % exp) time.sleep(60)
daemon=options.daemon if daemon not in daemon_types: print 'CRITICAL - ', daemon, 'is not a Shinken daemon!' parser.print_help() raise SystemExit(CRITICAL) port=options.portnum if port==0: port=daemon_types[daemon] con = None try: con = HTTPClient(address=options.hostname, port=port, timeout=options.timeout, data_timeout=options.data_timeout, use_ssl=options.ssl) result=con.get('ping') except Exception, exp: print "CRITICAL : the %s is not reachable : (%s)." % (daemon,exp) raise SystemExit(CRITICAL) if result: if result=='pong': if daemon != 'arbiter': try: result=con.get('have_conf') except Exception, exp: print "CRITICAL : the have_conf call to the %s failed : (%s)." % (daemon,exp) raise SystemExit(CRITICAL) if result:
class SatelliteLink(Item): """SatelliteLink is a common Class for link to satellite for Arbiter with Conf Dispatcher. """ # id = 0 each Class will have it's own id properties = Item.properties.copy() properties.update({ 'address': StringProp(fill_brok=['full_status']), 'timeout': IntegerProp(default='3', fill_brok=['full_status']), 'data_timeout': IntegerProp(default='120', fill_brok=['full_status']), 'check_interval': IntegerProp(default='60', fill_brok=['full_status']), 'max_check_attempts': IntegerProp(default='3', fill_brok=['full_status']), 'spare': BoolProp(default='0', fill_brok=['full_status']), 'manage_sub_realms': BoolProp(default='1', fill_brok=['full_status']), 'manage_arbiters': BoolProp(default='0', fill_brok=['full_status'], to_send=True), 'modules': ListProp(default='', to_send=True), 'polling_interval': IntegerProp(default='1', fill_brok=['full_status'], to_send=True), 'use_timezone': StringProp(default='NOTSET', to_send=True), 'realm': StringProp(default='', fill_brok=['full_status'], brok_transformation=get_obj_name_two_args_and_void), 'satellitemap': DictProp(default=None, elts_prop=AddrProp, to_send=True, override=True), 'use_ssl': BoolProp(default='0', fill_brok=['full_status']), }) running_properties = Item.running_properties.copy() running_properties.update({ 'con': StringProp(default=None), 'alive': StringProp(default=True, fill_brok=['full_status']), 'broks': StringProp(default=[]), 'attempt': StringProp(default=0, fill_brok=['full_status']), # the number of failed attempt 'reachable': StringProp(default=False, fill_brok=['full_status']), # can be network ask or not (dead or check in timeout or error) 'last_check': IntegerProp(default=0, fill_brok=['full_status']), 'managed_confs': StringProp(default={}), }) def __init__(self, *args, **kwargs): super(SatelliteLink, self).__init__(*args, **kwargs) self.arb_satmap = {'address': '0.0.0.0', 'port': 0} if hasattr(self, 'address'): self.arb_satmap['address'] = self.address if hasattr(self, 'port'): try: self.arb_satmap['port'] = int(self.port) except: pass def set_arbiter_satellitemap(self, satellitemap): """ arb_satmap is the satellitemap in current context: - A SatelliteLink is owned by an Arbiter - satellitemap attribute of SatelliteLink is the map defined IN THE satellite configuration but for creating connections, we need the have the satellitemap of the Arbiter """ self.arb_satmap = {'address': self.address, 'port': self.port} self.arb_satmap.update(satellitemap) def create_connection(self): self.con = HTTPClient(address=self.arb_satmap['address'], port=self.arb_satmap['port'], timeout=self.timeout, data_timeout=self.data_timeout, use_ssl=self.use_ssl ) self.uri = self.con.uri def put_conf(self, conf): if self.con is None: self.create_connection() # Maybe the connexion was not ok, bail out if not self.con: return False try: #pyro.set_timeout(self.con, self.data_timeout) self.con.post('put_conf', {'conf':conf}, wait='long') #pyro.set_timeout(self.con, self.timeout) print "PUT CONF SUCESS", self.get_name() return True except HTTPExceptions, exp: self.con = None logger.error("Failed sending configuration for %s: %s" % (self.get_name(), str(exp))) return False