def create_connection(self):
self.con = HTTPClient(address=self.arb_satmap['address'],
port=self.arb_satmap['port'],
timeout=self.timeout,
data_timeout=self.data_timeout,
use_ssl=self.use_ssl)
self.uri = self.con.uri
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
self.api_key = ''
self.secret = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
class Stats(object):
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
# There are two modes that are not exclusive
# first the kernel mode
self.api_key = ''
self.secret = ''
self.http_proxy = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
# then the statsd one
self.statsd_sock = None
self.statsd_addr = None
def launch_reaper_thread(self):
self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper')
self.reaper_thread.daemon = True
self.reaper_thread.start()
def register(self, app, name, _type, api_key='', secret='', http_proxy='',
statsd_host='localhost', statsd_port=8125, statsd_prefix='shinken',
statsd_enabled=False):
self.app = app
self.name = name
self.type = _type
# kernel.io part
self.api_key = api_key
self.secret = secret
self.http_proxy = http_proxy
# local statsd part
self.statsd_host = statsd_host
self.statsd_port = statsd_port
self.statsd_prefix = statsd_prefix
self.statsd_enabled = statsd_enabled
if self.statsd_enabled:
logger.debug('Loading statsd communication with %s:%s.%s',
self.statsd_host, self.statsd_port, self.statsd_prefix)
self.load_statsd()
# Also load the proxy if need
self.con.set_proxy(self.http_proxy)
# Let be crystal clear about why I don't use the statsd lib in python: it's crappy.
# how guys did you f**k this up to this point? django by default for the conf?? really?...
# So raw socket are far better here
def load_statsd(self):
try:
self.statsd_addr = (socket.gethostbyname(self.statsd_host), self.statsd_port)
self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
except (socket.error, socket.gaierror), exp:
logger.error('Cannot create statsd socket: %s' % exp)
return
class Stats(object):
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
# There are two modes that are not exclusive
# first the kernel mode
self.api_key = ''
self.secret = ''
self.http_proxy = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
# then the statsd one
self.statsd_sock = None
self.statsd_addr = None
def launch_reaper_thread(self):
self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper')
self.reaper_thread.daemon = True
self.reaper_thread.start()
def register(self, app, name, _type, api_key='', secret='', http_proxy='',
statsd_host='localhost', statsd_port=8125, statsd_prefix='shinken',
statsd_enabled=False):
self.app = app
self.name = name
self.type = _type
# kernel.io part
self.api_key = api_key
self.secret = secret
self.http_proxy = http_proxy
# local statsd part
self.statsd_host = statsd_host
self.statsd_port = statsd_port
self.statsd_prefix = statsd_prefix
self.statsd_enabled = statsd_enabled
if self.statsd_enabled:
logger.debug('Loading statsd communication with %s:%s.%s',
self.statsd_host, self.statsd_port, self.statsd_prefix)
self.load_statsd()
# Also load the proxy if need
self.con.set_proxy(self.http_proxy)
# Let be crystal clear about why I don't use the statsd lib in python: it's crappy.
# how guys did you f**k this up to this point? django by default for the conf?? really?...
# So raw socket are far better here
def load_statsd(self):
try:
self.statsd_addr = (socket.gethostbyname(self.statsd_host), self.statsd_port)
self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
except (socket.error, socket.gaierror), exp:
logger.error('Cannot create statsd socket: %s' % exp)
return
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
self.api_key = ''
self.secret = ''
self.cyph = None
self.con = HTTPClient(uri='http://metrology')
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
# There are two modes that are not exclusive
# first the kernel mode
self.api_key = ''
self.secret = ''
self.http_proxy = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
# then the statsd one
self.statsd_sock = None
self.statsd_addr = None
def pynag_con_init(self, id, type='scheduler'):
# Get the good links tab for looping..
links = self.get_links_from_type(type)
if links is None:
logger.debug('Type unknown for connection! %s' % type)
return
if type == 'scheduler':
# If sched is not active, I do not try to init
# it is just useless
is_active = links[id]['active']
if not is_active:
return
# If we try to connect too much, we slow down our tests
if self.is_connection_try_too_close(links[id]):
return
# Ok, we can now update it
links[id]['last_connection'] = time.time()
# DBG: print "Init connection with", links[id]['uri']
running_id = links[id]['running_id']
# DBG: print "Running id before connection", running_id
uri = links[id]['uri']
try:
con = links[id]['con'] = HTTPClient(
uri=uri, strong_ssl=links[id]['hard_ssl_name_check'])
except HTTPExceptions, exp:
# But the multiprocessing module is not compatible with it!
# so we must disable it immediately after
logger.info("Connection problem to the %s %s: %s" %
(type, links[id]['name'], str(exp)))
links[id]['con'] = None
return
def create_connection(self):
self.con = HTTPClient(address=self.arb_satmap['address'], port=self.arb_satmap['port'],
timeout=self.timeout, data_timeout=self.data_timeout,
use_ssl=self.use_ssl,
strong_ssl=self.hard_ssl_name_check
)
self.uri = self.con.uri
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
self.api_key = ''
self.secret = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
self.api_key = ''
self.secret = ''
self.cyph = None
self.con = HTTPClient(uri='http://metrology')
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
# There are two modes that are not exclusive
# first the kernel mode
self.api_key = ''
self.secret = ''
self.http_proxy = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
# then the statsd one
self.statsd_interval = 5
self.statsd_types = ['system', 'queue', 'perf']
self.statsd_sock = None
self.statsd_addr = None
self.statsd_types = 'system,object,queue,perf'
self.statsd_pattern = None
self.name_cache = {}
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
# There are two modes that are not exclusive
# first the kernel mode
self.api_key = ''
self.secret = ''
self.http_proxy = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
# then the statsd one
self.statsd_sock = None
self.statsd_addr = None
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
# There are two modes that are not exclusive
# first the kernel mode
self.api_key = ''
self.secret = ''
self.http_proxy = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
# then the statsd one
self.statsd_interval = 5
self.statsd_types = ['system', 'queue', 'perf']
self.statsd_sock = None
self.statsd_addr = None
self.statsd_types = 'system,object,queue,perf'
self.statsd_pattern = None
self.name_cache = {}
def __init__(self):
self.name = ""
self.type = ""
self.app = None
self.stats = {}
# There are two modes that are not exclusive
# first the kernel mode
self.api_key = ""
self.secret = ""
self.http_proxy = ""
self.con = HTTPClient(uri="http://kernel.shinken.io")
# then the statsd one
self.statsd_interval = 5
self.statsd_types = ["system", "queue", "perf"]
self.statsd_sock = None
self.statsd_addr = None
self.statsd_types = "system,object,queue,perf"
self.statsd_pattern = None
self.name_cache = {}
class Stats(object):
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
self.api_key = ''
self.secret = ''
self.cyph = None
self.con = HTTPClient(uri='http://metrology')
def launch_reaper_thread(self):
self.reaper_thread = threading.Thread(None,
target=self.reaper,
name='stats-reaper')
self.reaper_thread.daemon = True
self.reaper_thread.start()
def register(self, app, name, _type, api_key='', secret=''):
self.app = app
self.name = name
self.type = _type
self.api_key = api_key
self.secret = secret
# Assumea 16 len secret, but should be alreayd ok
self.secret += '\0' * (-len(self.secret) % 16)
if AES is not None and self.secret != '':
self.cyph = AES.new(self.secret, AES.MODE_ECB)
# Will increment a stat key, if None, start at 0
def incr(self, k, v):
_min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0))
nb += 1
_sum += v
if _min is None or v < _min:
_min = v
if _max is None or v > _max:
_max = v
self.stats[k] = (_min, _max, nb, _sum)
def reaper(self):
while True:
now = int(time.time())
logger.debug('REAPER loop')
stats = self.stats
self.stats = {}
if len(stats) != 0:
s = ', '.join(
['%s:%s' % (k, v) for (k, v) in stats.iteritems()])
logger.debug("REAPER: %s ", s)
# If we are not in an initializer daemon we skip, we cannot have a real name, it sucks
# to find the data after this
if not self.name:
time.sleep(10)
continue
logger.debug('REAPER we got a name')
metrics = []
for (k, e) in stats.iteritems():
nk = '%s.%s.%s' % (self.type, self.name, k)
logger.debug('REAP %s:%s', nk, e)
_min, _max, nb, _sum = e
_avg = float(_sum) / nb
# nb can't be 0 here and _min_max can't be None too
s = '%s.avg %f %d' % (nk, _avg, now)
metrics.append(s)
s = '%s.min %f %d' % (nk, _min, now)
metrics.append(s)
s = '%s.max %f %d' % (nk, _max, now)
metrics.append(s)
s = '%s.count %f %d' % (nk, nb, now)
metrics.append(s)
logger.debug('REAPER metrics to send %s (%d)', metrics,
len(str(metrics)))
# get the inner data for the daemon
struct = self.app.get_stats_struct()
struct['metrics'].extend(metrics)
logger.debug('REAPER whole struct %s', struct)
j = json.dumps(struct)
if self.cyph is not None:
logger.debug('PUT to /api/v1/put/ with %s %s', self.api_key,
self.secret)
# assume a %16 length messagexs
j += '\0' * (-len(j) % 16)
encrypted_text = self.cyph.encrypt(j)
try:
self.con.put('/api/v1/put/', encrypted_text)
except HTTPException, exp:
logger.debug('REAPER cannot put to the metric server %s',
exp)
time.sleep(10)
class Stats(object):
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
# There are two modes that are not exclusive
# first the kernel mode
self.api_key = ''
self.secret = ''
self.http_proxy = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
# then the statsd one
self.statsd_interval = 5
self.statsd_types = ['system', 'queue', 'perf']
self.statsd_sock = None
self.statsd_addr = None
self.statsd_types = 'system,object,queue,perf'
self.statsd_pattern = None
self.name_cache = {}
def launch_reaper_thread(self):
self.reaper_thread = threading.Thread(None,
target=self.reaper,
name='stats-reaper')
self.reaper_thread.daemon = True
self.reaper_thread.start()
def launch_harvester_thread(self):
self.harvester_thread = threading.Thread(None,
target=self.harvester,
name='stats-harvester')
self.harvester_thread.daemon = True
self.harvester_thread.start()
def register(self,
app,
name,
_type,
api_key='',
secret='',
http_proxy='',
statsd_host='localhost',
statsd_port=8125,
statsd_prefix='shinken',
statsd_enabled=False,
statsd_interval=5,
statsd_types=None,
statsd_pattern=None):
self.app = app
self.name = name
self.type = _type
# kernel.io part
self.api_key = api_key
self.secret = secret
self.http_proxy = http_proxy
# local statsd part
self.statsd_host = statsd_host
self.statsd_port = statsd_port
self.statsd_prefix = statsd_prefix
self.statsd_enabled = statsd_enabled
self.statsd_interval = statsd_interval
if statsd_types is not None:
self.statsd_types = [
t.strip() for t in statsd_types.split(",") if t.strip()
]
if statsd_pattern is not None:
self.statsd_pattern = statsd_pattern
self.name_cache = {}
if self.statsd_enabled:
logger.debug('Loading statsd communication with %s:%s.%s',
self.statsd_host, self.statsd_port,
self.statsd_prefix)
self.load_statsd()
# Also load the proxy if need
self.con.set_proxy(self.http_proxy)
# Tells whether statsd is enabled and stats should be sent
def is_statsd_enabled(self):
return (self.statsd_sock is not None and self.name
and self.app is not None)
# Tells whether kernel.status.io exporter is enabled
def is_shinkenio_enabled(self):
return (self.name and self.api_key and self.secret)
# Let be crystal clear about why I don't use the statsd lib in python: it's crappy.
# how guys did you f**k this up to this point? django by default for the conf?? really?...
# So raw socket are far better here
def load_statsd(self):
try:
self.statsd_addr = (socket.gethostbyname(self.statsd_host),
self.statsd_port)
self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
except (socket.error, socket.gaierror), exp:
logger.error('Cannot create statsd socket: %s' % exp)
return
class SatelliteLink(Item):
"""SatelliteLink is a common Class for link to satellite for
Arbiter with Conf Dispatcher.
"""
# id = 0 each Class will have it's own id
properties = Item.properties.copy()
properties.update({
'address':
StringProp(fill_brok=['full_status']),
'timeout':
IntegerProp(default='3', fill_brok=['full_status']),
'data_timeout':
IntegerProp(default='120', fill_brok=['full_status']),
'check_interval':
IntegerProp(default='60', fill_brok=['full_status']),
'max_check_attempts':
IntegerProp(default='3', fill_brok=['full_status']),
'spare':
BoolProp(default='0', fill_brok=['full_status']),
'manage_sub_realms':
BoolProp(default='1', fill_brok=['full_status']),
'manage_arbiters':
BoolProp(default='0', fill_brok=['full_status'], to_send=True),
'modules':
ListProp(default='', to_send=True),
'polling_interval':
IntegerProp(default='1', fill_brok=['full_status'], to_send=True),
'use_timezone':
StringProp(default='NOTSET', to_send=True),
'realm':
StringProp(default='',
fill_brok=['full_status'],
brok_transformation=get_obj_name_two_args_and_void),
'satellitemap':
DictProp(default=None, elts_prop=AddrProp, to_send=True,
override=True),
'use_ssl':
BoolProp(default='0', fill_brok=['full_status']),
})
running_properties = Item.running_properties.copy()
running_properties.update({
'con':
StringProp(default=None),
'alive':
StringProp(default=True, fill_brok=['full_status']),
'broks':
StringProp(default=[]),
'attempt':
StringProp(default=0,
fill_brok=['full_status']), # the number of failed attempt
'reachable':
StringProp(default=False, fill_brok=[
'full_status'
]), # can be network ask or not (dead or check in timeout or error)
'last_check':
IntegerProp(default=0, fill_brok=['full_status']),
'managed_confs':
StringProp(default={}),
})
def __init__(self, *args, **kwargs):
super(SatelliteLink, self).__init__(*args, **kwargs)
self.arb_satmap = {'address': '0.0.0.0', 'port': 0}
if hasattr(self, 'address'):
self.arb_satmap['address'] = self.address
if hasattr(self, 'port'):
try:
self.arb_satmap['port'] = int(self.port)
except:
pass
def set_arbiter_satellitemap(self, satellitemap):
"""
arb_satmap is the satellitemap in current context:
- A SatelliteLink is owned by an Arbiter
- satellitemap attribute of SatelliteLink is the map defined IN THE satellite configuration
but for creating connections, we need the have the satellitemap of the Arbiter
"""
self.arb_satmap = {'address': self.address, 'port': self.port}
self.arb_satmap.update(satellitemap)
def create_connection(self):
self.con = HTTPClient(address=self.arb_satmap['address'],
port=self.arb_satmap['port'],
timeout=self.timeout,
data_timeout=self.data_timeout,
use_ssl=self.use_ssl)
self.uri = self.con.uri
def put_conf(self, conf):
if self.con is None:
self.create_connection()
# Maybe the connexion was not ok, bail out
if not self.con:
return False
try:
#pyro.set_timeout(self.con, self.data_timeout)
self.con.post('put_conf', {'conf': conf}, wait='long')
#pyro.set_timeout(self.con, self.timeout)
print "PUT CONF SUCESS", self.get_name()
return True
except HTTPExceptions, exp:
self.con = None
logger.error("Failed sending configuration for %s: %s" %
(self.get_name(), str(exp)))
return False
class Stats(object):
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
self.api_key = ''
self.secret = ''
self.cyph = None
self.con = HTTPClient(uri='http://metrology')
def launch_reaper_thread(self):
self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper')
self.reaper_thread.daemon = True
self.reaper_thread.start()
def register(self, app, name, _type, api_key='', secret=''):
self.app = app
self.name = name
self.type = _type
self.api_key = api_key
self.secret = secret
# Assumea 16 len secret, but should be alreayd ok
self.secret += '\0' * (-len(self.secret) % 16)
if AES is not None and self.secret != '':
self.cyph = AES.new(self.secret, AES.MODE_ECB)
# Will increment a stat key, if None, start at 0
def incr(self, k, v):
_min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0))
nb += 1
_sum += v
if _min is None or v < _min:
_min = v
if _max is None or v > _max:
_max = v
self.stats[k] = (_min, _max, nb, _sum)
def reaper(self):
while True:
now = int(time.time())
logger.debug('REAPER loop')
stats = self.stats
self.stats = {}
if len(stats) != 0:
s = ', '.join(['%s:%s' % (k,v) for (k,v) in stats.iteritems()])
logger.debug("REAPER: %s ", s)
# If we are not in an initializer daemon we skip, we cannot have a real name, it sucks
# to find the data after this
if not self.name:
time.sleep(10)
continue
logger.debug('REAPER we got a name')
metrics = []
for (k,e) in stats.iteritems():
nk = '%s.%s.%s' % (self.type, self.name, k)
logger.debug('REAP %s:%s', nk, e)
_min, _max, nb, _sum = e
_avg = float(_sum) / nb
# nb can't be 0 here and _min_max can't be None too
s = '%s.avg %f %d' % (nk, _avg, now)
metrics.append(s)
s = '%s.min %f %d' % (nk, _min, now)
metrics.append(s)
s = '%s.max %f %d' % (nk, _max, now)
metrics.append(s)
s = '%s.count %f %d' % (nk, nb, now)
metrics.append(s)
logger.debug('REAPER metrics to send %s (%d)', metrics, len(str(metrics)) )
# get the inner data for the daemon
struct = self.app.get_stats_struct()
struct['metrics'].extend(metrics)
logger.debug('REAPER whole struct %s', struct)
j = json.dumps(struct)
if self.cyph is not None:
logger.debug('PUT to /api/v1/put/ with %s %s', self.api_key, self.secret)
# assume a %16 length messagexs
j += '\0' * (-len(j) % 16)
encrypted_text = self.cyph.encrypt(j)
try:
self.con.put('/api/v1/put/', encrypted_text)
except HTTPException, exp:
logger.debug('REAPER cannot put to the metric server %s', exp)
time.sleep(10)
class Stats(object):
def __init__(self):
self.name = ""
self.type = ""
self.app = None
self.stats = {}
# There are two modes that are not exclusive
# first the kernel mode
self.api_key = ""
self.secret = ""
self.http_proxy = ""
self.con = HTTPClient(uri="http://kernel.shinken.io")
# then the statsd one
self.statsd_interval = 5
self.statsd_types = ["system", "queue", "perf"]
self.statsd_sock = None
self.statsd_addr = None
self.statsd_types = "system,object,queue,perf"
self.statsd_pattern = None
self.name_cache = {}
def launch_reaper_thread(self):
self.reaper_thread = threading.Thread(None, target=self.reaper, name="stats-reaper")
self.reaper_thread.daemon = True
self.reaper_thread.start()
def launch_harvester_thread(self):
self.harvester_thread = threading.Thread(None, target=self.harvester, name="stats-harvester")
self.harvester_thread.daemon = True
self.harvester_thread.start()
def register(
self,
app,
name,
_type,
api_key="",
secret="",
http_proxy="",
statsd_host="localhost",
statsd_port=8125,
statsd_prefix="shinken",
statsd_enabled=False,
statsd_interval=5,
statsd_types=None,
statsd_pattern=None,
):
self.app = app
self.name = name
self.type = _type
# kernel.io part
self.api_key = api_key
self.secret = secret
self.http_proxy = http_proxy
# local statsd part
self.statsd_host = statsd_host
self.statsd_port = statsd_port
self.statsd_prefix = statsd_prefix
self.statsd_enabled = statsd_enabled
self.statsd_interval = statsd_interval
if statsd_types is not None:
self.statsd_types = [t.strip() for t in statsd_types.split(",") if t.strip()]
if statsd_pattern is not None:
self.statsd_pattern = statsd_pattern
self.name_cache = {}
if self.statsd_enabled:
logger.debug(
"Loading statsd communication with %s:%s.%s", self.statsd_host, self.statsd_port, self.statsd_prefix
)
self.load_statsd()
# Also load the proxy if need
self.con.set_proxy(self.http_proxy)
# Tells whether statsd is enabled and stats should be sent
def is_statsd_enabled(self):
return self.statsd_sock is not None and self.name and self.app is not None
# Tells whether kernel.status.io exporter is enabled
def is_shinkenio_enabled(self):
return self.name and self.api_key and self.secret
# Let be crystal clear about why I don't use the statsd lib in python: it's crappy.
# how guys did you f**k this up to this point? django by default for the conf?? really?...
# So raw socket are far better here
def load_statsd(self):
try:
self.statsd_addr = (socket.gethostbyname(self.statsd_host), self.statsd_port)
self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
except (socket.error, socket.gaierror), exp:
logger.error("Cannot create statsd socket: %s" % exp)
return
class SatelliteLink(Item):
"""SatelliteLink is a common Class for link to satellite for
Arbiter with Conf Dispatcher.
"""
# id = 0 each Class will have it's own id
properties = Item.properties.copy()
properties.update(
{
"address": StringProp(fill_brok=["full_status"]),
"timeout": IntegerProp(default="3", fill_brok=["full_status"]),
"data_timeout": IntegerProp(default="120", fill_brok=["full_status"]),
"check_interval": IntegerProp(default="60", fill_brok=["full_status"]),
"max_check_attempts": IntegerProp(default="3", fill_brok=["full_status"]),
"spare": BoolProp(default="0", fill_brok=["full_status"]),
"manage_sub_realms": BoolProp(default="1", fill_brok=["full_status"]),
"manage_arbiters": BoolProp(default="0", fill_brok=["full_status"], to_send=True),
"modules": ListProp(default="", to_send=True),
"polling_interval": IntegerProp(default="1", fill_brok=["full_status"], to_send=True),
"use_timezone": StringProp(default="NOTSET", to_send=True),
"realm": StringProp(
default="", fill_brok=["full_status"], brok_transformation=get_obj_name_two_args_and_void
),
"satellitemap": DictProp(default=None, elts_prop=AddrProp, to_send=True, override=True),
"use_ssl": BoolProp(default="0", fill_brok=["full_status"]),
"hard_ssl_name_check": BoolProp(default="0", fill_brok=["full_status"]),
"passive": BoolProp(default="0", fill_brok=["full_status"], to_send=True),
}
)
running_properties = Item.running_properties.copy()
running_properties.update(
{
"con": StringProp(default=None),
"alive": StringProp(default=True, fill_brok=["full_status"]),
"broks": StringProp(default=[]),
"attempt": StringProp(default=0, fill_brok=["full_status"]), # the number of failed attempt
"reachable": StringProp(
default=False, fill_brok=["full_status"]
), # can be network ask or not (dead or check in timeout or error)
"last_check": IntegerProp(default=0, fill_brok=["full_status"]),
"managed_confs": StringProp(default={}),
}
)
def __init__(self, *args, **kwargs):
super(SatelliteLink, self).__init__(*args, **kwargs)
self.arb_satmap = {"address": "0.0.0.0", "port": 0}
if hasattr(self, "address"):
self.arb_satmap["address"] = self.address
if hasattr(self, "port"):
try:
self.arb_satmap["port"] = int(self.port)
except:
pass
def set_arbiter_satellitemap(self, satellitemap):
"""
arb_satmap is the satellitemap in current context:
- A SatelliteLink is owned by an Arbiter
- satellitemap attribute of SatelliteLink is the map defined IN THE satellite configuration
but for creating connections, we need the have the satellitemap of the Arbiter
"""
self.arb_satmap = {
"address": self.address,
"port": self.port,
"use_ssl": self.use_ssl,
"hard_ssl_name_check": self.hard_ssl_name_check,
}
self.arb_satmap.update(satellitemap)
def create_connection(self):
self.con = HTTPClient(
address=self.arb_satmap["address"],
port=self.arb_satmap["port"],
timeout=self.timeout,
data_timeout=self.data_timeout,
use_ssl=self.use_ssl,
strong_ssl=self.hard_ssl_name_check,
)
self.uri = self.con.uri
def put_conf(self, conf):
if self.con is None:
self.create_connection()
# Maybe the connexion was not ok, bail out
if not self.con:
return False
try:
# pyro.set_timeout(self.con, self.data_timeout)
self.con.get("ping")
self.con.post("put_conf", {"conf": conf}, wait="long")
# pyro.set_timeout(self.con, self.timeout)
print "PUT CONF SUCESS", self.get_name()
return True
except HTTPExceptions, exp:
self.con = None
logger.error("Failed sending configuration for %s: %s" % (self.get_name(), str(exp)))
return False
class Stats(object):
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
self.api_key = ''
self.secret = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
def launch_reaper_thread(self):
self.reaper_thread = threading.Thread(None,
target=self.reaper,
name='stats-reaper')
self.reaper_thread.daemon = True
self.reaper_thread.start()
def register(self, app, name, _type, api_key='', secret=''):
self.app = app
self.name = name
self.type = _type
self.api_key = api_key
self.secret = secret
# Will increment a stat key, if None, start at 0
def incr(self, k, v):
_min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0))
nb += 1
_sum += v
if _min is None or v < _min:
_min = v
if _max is None or v > _max:
_max = v
self.stats[k] = (_min, _max, nb, _sum)
def _encrypt(self, data):
m = hashlib.md5()
m.update(self.secret)
key = m.hexdigest()
m = hashlib.md5()
m.update(self.secret + key)
iv = m.hexdigest()
data = pad(data)
aes = AES.new(key, AES.MODE_CBC, iv[:16])
encrypted = aes.encrypt(data)
return base64.urlsafe_b64encode(encrypted)
def reaper(self):
while True:
now = int(time.time())
stats = self.stats
self.stats = {}
if len(stats) != 0:
s = ', '.join(
['%s:%s' % (k, v) for (k, v) in stats.iteritems()])
# If we are not in an initializer daemon we skip, we cannot have a real name, it sucks
# to find the data after this
if not self.name or not self.api_key or not self.secret:
time.sleep(60)
continue
metrics = []
for (k, e) in stats.iteritems():
nk = '%s.%s.%s' % (self.type, self.name, k)
_min, _max, nb, _sum = e
_avg = float(_sum) / nb
# nb can't be 0 here and _min_max can't be None too
s = '%s.avg %f %d' % (nk, _avg, now)
metrics.append(s)
s = '%s.min %f %d' % (nk, _min, now)
metrics.append(s)
s = '%s.max %f %d' % (nk, _max, now)
metrics.append(s)
s = '%s.count %f %d' % (nk, nb, now)
metrics.append(s)
#logger.debug('REAPER metrics to send %s (%d)' % (metrics, len(str(metrics))) )
# get the inner data for the daemon
struct = self.app.get_stats_struct()
struct['metrics'].extend(metrics)
#logger.debug('REAPER whole struct %s' % struct)
j = json.dumps(struct)
if AES is not None and self.secret != '':
# RESET AFTER EACH calls!
logger.debug(
'Stats PUT to kernel.shinken.io/api/v1/put/ with %s %s' %
(self.api_key, self.secret))
# assume a %16 length messagexs
encrypted_text = self._encrypt(j)
try:
r = self.con.put(
'/api/v1/put/?api_key=%s' % (self.api_key),
encrypted_text)
except HTTPException, exp:
logger.debug(
'Stats REAPER cannot put to the metric server %s' %
exp)
time.sleep(60)
class Stats(object):
def __init__(self):
self.name = ''
self.type = ''
self.app = None
self.stats = {}
self.api_key = ''
self.secret = ''
self.con = HTTPClient(uri='http://kernel.shinken.io')
def launch_reaper_thread(self):
self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper')
self.reaper_thread.daemon = True
self.reaper_thread.start()
def register(self, app, name, _type, api_key='', secret=''):
self.app = app
self.name = name
self.type = _type
self.api_key = api_key
self.secret = secret
# Will increment a stat key, if None, start at 0
def incr(self, k, v):
_min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0))
nb += 1
_sum += v
if _min is None or v < _min:
_min = v
if _max is None or v > _max:
_max = v
self.stats[k] = (_min, _max, nb, _sum)
def _encrypt(self, data):
m = hashlib.md5()
m.update(self.secret)
key = m.hexdigest()
m = hashlib.md5()
m.update(self.secret + key)
iv = m.hexdigest()
data = pad(data)
aes = AES.new(key, AES.MODE_CBC, iv[:16])
encrypted = aes.encrypt(data)
return base64.urlsafe_b64encode(encrypted)
def reaper(self):
while True:
now = int(time.time())
stats = self.stats
self.stats = {}
if len(stats) != 0:
s = ', '.join(['%s:%s' % (k,v) for (k,v) in stats.iteritems()])
# If we are not in an initializer daemon we skip, we cannot have a real name, it sucks
# to find the data after this
if not self.name or not self.api_key or not self.secret:
time.sleep(60)
continue
metrics = []
for (k,e) in stats.iteritems():
nk = '%s.%s.%s' % (self.type, self.name, k)
_min, _max, nb, _sum = e
_avg = float(_sum) / nb
# nb can't be 0 here and _min_max can't be None too
s = '%s.avg %f %d' % (nk, _avg, now)
metrics.append(s)
s = '%s.min %f %d' % (nk, _min, now)
metrics.append(s)
s = '%s.max %f %d' % (nk, _max, now)
metrics.append(s)
s = '%s.count %f %d' % (nk, nb, now)
metrics.append(s)
#logger.debug('REAPER metrics to send %s (%d)' % (metrics, len(str(metrics))) )
# get the inner data for the daemon
struct = self.app.get_stats_struct()
struct['metrics'].extend(metrics)
#logger.debug('REAPER whole struct %s' % struct)
j = json.dumps(struct)
if AES is not None and self.secret != '':
# RESET AFTER EACH calls!
logger.debug('Stats PUT to kernel.shinken.io/api/v1/put/ with %s %s' % (self.api_key, self.secret))
# assume a %16 length messagexs
encrypted_text = self._encrypt(j)
try:
r = self.con.put('/api/v1/put/?api_key=%s' % (self.api_key), encrypted_text)
except HTTPException, exp:
logger.debug('Stats REAPER cannot put to the metric server %s' % exp)
time.sleep(60)
daemon=options.daemon
if daemon not in daemon_types:
print 'CRITICAL - ', daemon, 'is not a Shinken daemon!'
parser.print_help()
raise SystemExit(CRITICAL)
port=options.portnum
if port==0:
port=daemon_types[daemon]
con = None
try:
con = HTTPClient(address=options.hostname, port=port, timeout=options.timeout, data_timeout=options.data_timeout, use_ssl=options.ssl)
result=con.get('ping')
except Exception, exp:
print "CRITICAL : the %s is not reachable : (%s)." % (daemon,exp)
raise SystemExit(CRITICAL)
if result:
if result=='pong':
if daemon != 'arbiter':
try:
result=con.get('have_conf')
except Exception, exp:
print "CRITICAL : the have_conf call to the %s failed : (%s)." % (daemon,exp)
raise SystemExit(CRITICAL)
if result:
class SatelliteLink(Item):
"""SatelliteLink is a common Class for link to satellite for
Arbiter with Conf Dispatcher.
"""
# id = 0 each Class will have it's own id
properties = Item.properties.copy()
properties.update({
'address': StringProp(fill_brok=['full_status']),
'timeout': IntegerProp(default='3', fill_brok=['full_status']),
'data_timeout': IntegerProp(default='120', fill_brok=['full_status']),
'check_interval': IntegerProp(default='60', fill_brok=['full_status']),
'max_check_attempts': IntegerProp(default='3', fill_brok=['full_status']),
'spare': BoolProp(default='0', fill_brok=['full_status']),
'manage_sub_realms': BoolProp(default='1', fill_brok=['full_status']),
'manage_arbiters': BoolProp(default='0', fill_brok=['full_status'], to_send=True),
'modules': ListProp(default='', to_send=True),
'polling_interval': IntegerProp(default='1', fill_brok=['full_status'], to_send=True),
'use_timezone': StringProp(default='NOTSET', to_send=True),
'realm': StringProp(default='', fill_brok=['full_status'], brok_transformation=get_obj_name_two_args_and_void),
'satellitemap': DictProp(default=None, elts_prop=AddrProp, to_send=True, override=True),
'use_ssl': BoolProp(default='0', fill_brok=['full_status']),
})
running_properties = Item.running_properties.copy()
running_properties.update({
'con': StringProp(default=None),
'alive': StringProp(default=True, fill_brok=['full_status']),
'broks': StringProp(default=[]),
'attempt': StringProp(default=0, fill_brok=['full_status']), # the number of failed attempt
'reachable': StringProp(default=False, fill_brok=['full_status']), # can be network ask or not (dead or check in timeout or error)
'last_check': IntegerProp(default=0, fill_brok=['full_status']),
'managed_confs': StringProp(default={}),
})
def __init__(self, *args, **kwargs):
super(SatelliteLink, self).__init__(*args, **kwargs)
self.arb_satmap = {'address': '0.0.0.0', 'port': 0}
if hasattr(self, 'address'):
self.arb_satmap['address'] = self.address
if hasattr(self, 'port'):
try:
self.arb_satmap['port'] = int(self.port)
except:
pass
def set_arbiter_satellitemap(self, satellitemap):
"""
arb_satmap is the satellitemap in current context:
- A SatelliteLink is owned by an Arbiter
- satellitemap attribute of SatelliteLink is the map defined IN THE satellite configuration
but for creating connections, we need the have the satellitemap of the Arbiter
"""
self.arb_satmap = {'address': self.address, 'port': self.port}
self.arb_satmap.update(satellitemap)
def create_connection(self):
self.con = HTTPClient(address=self.arb_satmap['address'], port=self.arb_satmap['port'],
timeout=self.timeout, data_timeout=self.data_timeout, use_ssl=self.use_ssl
)
self.uri = self.con.uri
def put_conf(self, conf):
if self.con is None:
self.create_connection()
# Maybe the connexion was not ok, bail out
if not self.con:
return False
try:
#pyro.set_timeout(self.con, self.data_timeout)
self.con.post('put_conf', {'conf':conf}, wait='long')
#pyro.set_timeout(self.con, self.timeout)
print "PUT CONF SUCESS", self.get_name()
return True
except HTTPExceptions, exp:
self.con = None
logger.error("Failed sending configuration for %s: %s" % (self.get_name(), str(exp)))
return False
daemon=options.daemon
if daemon not in daemon_types:
print 'CRITICAL - ', daemon, 'is not a Shinken daemon!'
parser.print_help()
raise SystemExit(CRITICAL)
port=options.portnum
if port==0:
port=daemon_types[daemon]
con = None
try:
con = HTTPClient(address=options.hostname, port=port, timeout=options.timeout, data_timeout=options.data_timeout, use_ssl=options.ssl)
result=con.get('ping')
except Exception, exp:
print "CRITICAL : the %s is not reachable : (%s)." % (daemon,exp)
raise SystemExit(CRITICAL)
if result:
if result=='pong':
if daemon != 'arbiter':
try:
result=con.get('have_conf')
except Exception, exp:
print "CRITICAL : the have_conf call to the %s failed : (%s)." % (daemon,exp)
raise SystemExit(CRITICAL)
if result:
