def on_updated_host(updated, original):
"""
What to do when an host live state is updated ...
If the host monitored state is changing, recalculate the live synthesis, else
simply update the livesynthesis
"""
if original['_is_template']:
return
# If the host is not monitored and we do not change its monitoring state
if not original['active_checks_enabled'] and not original['passive_checks_enabled'] \
and 'active_checks_enabled' not in updated \
and 'passive_checks_enabled' not in updated:
return
minus, plus = Livesynthesis.livesynthesis_to_update('hosts', updated, original)
if minus is not False:
livesynthesis_db = current_app.data.driver.db['livesynthesis']
live_current = livesynthesis_db.find_one({'_realm': original['_realm']})
if live_current is None or 'not_monitored' in minus \
or (plus and 'not_monitored' in plus):
ls = Livesynthesis()
ls.recalculate()
else:
data = {"$inc": {minus: -1}}
if plus is not False:
data = {"$inc": {minus: -1, plus: 1}}
current_app.logger.debug("LS - updated host %s: %s...", original['name'], data)
current_app.data.driver.db.livesynthesis.update({'_id': live_current['_id']}, data)
# Send livesynthesis to TSDB
live_current = livesynthesis_db.find_one({'_realm': original['_realm']})
Timeseries.send_livesynthesis_metrics(original['_realm'], live_current)
def on_inserted_service(items):
"""
What to do when a service is inserted in the backend ...
"""
livesynthesis_db = current_app.data.driver.db['livesynthesis']
for _, item in enumerate(items):
if item['_is_template']:
continue
live_current = livesynthesis_db.find_one({'_realm': item['_realm']})
if live_current is None:
ls = Livesynthesis()
ls.recalculate()
else:
typecheck = 'services'
if not item['active_checks_enabled'] and not item['passive_checks_enabled']:
data = {"$inc": {"%s_not_monitored" % typecheck: 1,
"%s_total" % typecheck: 1}}
else:
data = {"$inc": {"%s_%s_%s" % (typecheck, item['ls_state'].lower(),
item['ls_state_type'].lower()): 1,
"%s_total" % typecheck: 1}}
current_app.logger.debug("LS - inserted service %s: %s...", item['name'], data)
current_app.data.driver.db.livesynthesis.update({'_id': live_current['_id']}, data)
# Send livesynthesis to TSDB
live_current = livesynthesis_db.find_one({'_realm': item['_realm']})
Timeseries.send_livesynthesis_metrics(item['_realm'], live_current)
def cron_timeseries():
"""
Cron used to add perfdata from retention to timeseries databases
:return: None
"""
with app.test_request_context():
timeseriesretention_db = current_app.data.driver.db[
'timeseriesretention']
if timeseriesretention_db.find().count() > 0:
tsc = timeseriesretention_db.find({
'for_graphite': True,
'for_influxdb': False
})
for data in tsc:
if not Timeseries.send_to_timeseries_graphite([data]):
break
lookup = {"_id": data['_id']}
deleteitem_internal('timeseriesretention', False, False,
**lookup)
tsc = timeseriesretention_db.find({
'for_graphite': False,
'for_influxdb': True
})
for data in tsc:
if not Timeseries.send_to_timeseries_influxdb([data]):
break
lookup = {"_id": data['_id']}
deleteitem_internal('timeseriesretention', False, False,
**lookup)
tsc = timeseriesretention_db.find({
'for_graphite': True,
'for_influxdb': True
})
for data in tsc:
graphite_serv = True
influxdb_serv = True
if not Timeseries.send_to_timeseries_graphite([data]):
graphite_serv = False
if not Timeseries.send_to_timeseries_influxdb([data]):
influxdb_serv = False
lookup = {"_id": data['_id']}
if graphite_serv and influxdb_serv:
deleteitem_internal('timeseriesretention', False, False,
**lookup)
elif graphite_serv and not influxdb_serv:
patch_internal('timeseriesretention',
{"for_graphite": False}, False, False,
**lookup)
elif influxdb_serv and not graphite_serv:
patch_internal('timeseriesretention',
{"for_influxdb": False}, False, False,
**lookup)
def cron_timeseries():
"""
Cron used to add perfdata from retention to timeseries databases
:return: None
"""
with app.test_request_context():
timeseriesretention_db = current_app.data.driver.db['timeseriesretention']
if timeseriesretention_db.find().count() > 0:
tsc = timeseriesretention_db.find({'for_graphite': True, 'for_influxdb': False})
for data in tsc:
if not Timeseries.send_to_timeseries_graphite([data]):
break
lookup = {"_id": data['_id']}
deleteitem_internal('timeseriesretention', False, False, **lookup)
tsc = timeseriesretention_db.find({'for_graphite': False, 'for_influxdb': True})
for data in tsc:
if not Timeseries.send_to_timeseries_influxdb([data]):
break
lookup = {"_id": data['_id']}
deleteitem_internal('timeseriesretention', False, False, **lookup)
tsc = timeseriesretention_db.find({'for_graphite': True, 'for_influxdb': True})
for data in tsc:
graphite_serv = True
influxdb_serv = True
if not Timeseries.send_to_timeseries_graphite([data]):
graphite_serv = False
if not Timeseries.send_to_timeseries_influxdb([data]):
influxdb_serv = False
lookup = {"_id": data['_id']}
if graphite_serv and influxdb_serv:
deleteitem_internal('timeseriesretention', False, False, **lookup)
elif graphite_serv and not influxdb_serv:
patch_internal('timeseriesretention', {"for_graphite": False}, False, False,
**lookup)
elif influxdb_serv and not graphite_serv:
patch_internal('timeseriesretention', {"for_influxdb": False}, False, False,
**lookup)
def on_deleted_service(item):
"""When delete a service, decrement livesynthesis
:param item: fields of the item deleted
:type item: dict
:return: None
"""
if item['_is_template']:
return
livesynthesis_db = current_app.data.driver.db['livesynthesis']
live_current = livesynthesis_db.find_one({'_realm': item['_realm']})
if live_current is None:
ls = Livesynthesis()
ls.recalculate()
else:
minus = Livesynthesis.livesynthesis_to_delete('services', item)
data = {"$inc": {minus: -1, 'services_total': -1}}
current_app.logger.debug("LS - Deleted service %s: %s", item['name'], data)
current_app.data.driver.db.livesynthesis.update({'_id': live_current['_id']}, data)
# Send livesynthesis to TSDB
live_current = livesynthesis_db.find_one({'_realm': item['_realm']})
Timeseries.send_livesynthesis_metrics(item['_realm'], live_current)
def test_prepare_data(self):
"""
Prepare timeseries from a web perfdata
:return: None
"""
item = {
'host': 'srv001',
'service': 'check_xxx',
'state': 'OK',
'state_type': 'HARD',
'state_id': 0,
'acknowledged': False,
'last_check': int(time.time()),
'last_state': 'OK',
'output': 'NGINX OK - 0.161 sec. response time, Active: 25 (Writing: 3 Reading: 0 '
'Waiting: 22) ReqPerSec: 58.000 ConnPerSec: 1.200 ReqPerConn: 4.466',
'long_output': '',
'perf_data': 'Writing=3;;;; Reading=0;;;; Waiting=22;;;; Active=25;1000;2000;; '
'ReqPerSec=58.000000;100;200;; ConnPerSec=1.200000;200;300;; '
'ReqPerConn=4.465602;;;;',
'_realm': 'All.Propieres'
}
ret = Timeseries.prepare_data(item)
reference = {
'data': [
{
'name': 'ReqPerConn',
'value': {
'name': 'ReqPerConn',
'min': None,
'max': None,
'value': 4.465602,
'warning': None,
'critical': None,
'uom': ''
}
},
{
'name': 'Writing',
'value': {
'name': 'Writing',
'min': None,
'max': None,
'value': 3,
'warning': None,
'critical': None,
'uom': ''
}
},
{
'name': 'Waiting',
'value': {
'name': 'Waiting',
'min': None,
'max': None,
'value': 22,
'warning': None,
'critical': None,
'uom': ''
}
},
{
'name': 'ConnPerSec',
'value': {
'name': 'ConnPerSec',
'min': None,
'max': None,
'value': 1.2,
'warning': 200,
'critical': 300,
'uom': ''
}
},
{
'name': 'Active',
'value': {
'name': 'Active',
'min': None,
'max': None,
'value': 25,
'warning': 1000,
'critical': 2000,
'uom': ''
}
},
{
'name': 'ReqPerSec',
'value': {
'name': 'ReqPerSec',
'min': None,
'max': None,
'value': 58,
'warning': 100,
'critical': 200,
'uom': ''
}
},
{
'name': 'Reading',
'value': {
'name': 'Reading',
'min': None,
'max': None,
'value': 0,
'warning': None,
'critical': None,
'uom': ''
}
}
]
}
self.assertItemsEqual(reference, ret)
def recalculate():
"""
Recalculate all the live synthesis counters
"""
current_app.logger.debug("LS - Recalculating...")
livesynthesis = current_app.data.driver.db['livesynthesis']
realmsdrv = current_app.data.driver.db['realm']
allrealms = realmsdrv.find()
for _, realm in enumerate(allrealms):
live_current = livesynthesis.find_one({'_realm': realm['_id']})
if live_current is None:
current_app.logger.debug(" new LS for realm %s", realm['name'])
data = {
'hosts_total': 0,
'hosts_not_monitored': 0,
'hosts_up_hard': 0,
'hosts_up_soft': 0,
'hosts_down_hard': 0,
'hosts_down_soft': 0,
'hosts_unreachable_hard': 0,
'hosts_unreachable_soft': 0,
'hosts_acknowledged': 0,
'hosts_in_downtime': 0,
'hosts_flapping': 0,
'services_total': 0,
'services_not_monitored': 0,
'services_ok_hard': 0,
'services_ok_soft': 0,
'services_warning_hard': 0,
'services_warning_soft': 0,
'services_critical_hard': 0,
'services_critical_soft': 0,
'services_unreachable_hard': 0,
'services_unreachable_soft': 0,
'services_unknown_hard': 0,
'services_unknown_soft': 0,
'services_acknowledged': 0,
'services_in_downtime': 0,
'services_flapping': 0,
'_realm': realm['_id']
}
livesynthesis.insert(data)
live_current = livesynthesis.find_one({'_realm': realm['_id']})
# Update hosts live synthesis
hosts = current_app.data.driver.db['host']
hosts_count = hosts.count({'_is_template': False, '_realm': realm['_id']})
data = {'hosts_total': hosts_count}
data['hosts_not_monitored'] = hosts.count({
'_is_template': False,
'active_checks_enabled': False, 'passive_checks_enabled': False,
'_realm': realm['_id']
})
data['hosts_up_hard'] = hosts.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'UP', 'ls_state_type': 'HARD',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['hosts_down_hard'] = hosts.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'DOWN', 'ls_state_type': 'HARD',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['hosts_unreachable_hard'] = hosts.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'UNREACHABLE', 'ls_state_type': 'HARD',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['hosts_up_soft'] = hosts.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'UP', 'ls_state_type': 'SOFT',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['hosts_down_soft'] = hosts.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'DOWN', 'ls_state_type': 'SOFT',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['hosts_unreachable_soft'] = hosts.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'UNREACHABLE', 'ls_state_type': 'SOFT',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['hosts_acknowledged'] = hosts.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_acknowledged': True, '_realm': realm['_id']
})
data['hosts_in_downtime'] = hosts.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_downtimed': True, '_realm': realm['_id']
})
current_app.logger.debug(" realm %s, hosts LS: %s", realm['name'], data)
lookup = {"_id": live_current['_id']}
patch_internal('livesynthesis', data, False, False, **lookup)
# Send livesynthesis to TSDB
Timeseries.send_livesynthesis_metrics(realm['_id'], data)
# Update services live synthesis
services = current_app.data.driver.db['service']
services_count = services.count({'_is_template': False, '_realm': realm['_id']})
data = {'services_total': services_count}
data['services_not_monitored'] = services.count({
'_is_template': False,
'active_checks_enabled': False, 'passive_checks_enabled': False,
'_realm': realm['_id']
})
data['services_ok_hard'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'OK', 'ls_state_type': 'HARD',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['services_warning_hard'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'WARNING', 'ls_state_type': 'HARD',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['services_critical_hard'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'CRITICAL', 'ls_state_type': 'HARD',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['services_unknown_hard'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'UNKNOWN', 'ls_state_type': 'HARD',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['services_unreachable_hard'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'UNREACHABLE', 'ls_state_type': 'HARD',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['services_ok_soft'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'OK', 'ls_state_type': 'SOFT',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['services_warning_soft'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'WARNING', 'ls_state_type': 'SOFT',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['services_critical_soft'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'CRITICAL', 'ls_state_type': 'SOFT',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['services_unknown_soft'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'UNKNOWN', 'ls_state_type': 'SOFT',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['services_unreachable_soft'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_state': 'UNREACHABLE', 'ls_state_type': 'SOFT',
'ls_acknowledged': False, '_realm': realm['_id']
})
data['services_acknowledged'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_acknowledged': True, '_realm': realm['_id']
})
data['services_in_downtime'] = services.count({
'_is_template': False,
'$or': [{'active_checks_enabled': True}, {'passive_checks_enabled': True}],
'ls_downtimed': True, '_realm': realm['_id']
})
current_app.logger.debug(" realm %s, services LS: %s", realm['name'], data)
lookup = {"_id": live_current['_id']}
patch_internal('livesynthesis', data, False, False, **lookup)
# Send livesynthesis to TSDB
Timeseries.send_livesynthesis_metrics(realm['_id'], data)
def build_target(self, item, fields):
"""
:param item: concerned host or service
:type item: dict
:param fields: fields of the concerned metric:
{'name': u'uptime_minutes',
'min': None, 'max': None,
'value': 92348,
'warning': None, 'critical': None,
'uom': u''}
:type fields: dict
:return:
"""
# Find the TS corresponding to the host realm
ts = self.timeseries[item['_realm']]
# Add statsd, graphite and realm prefixes
my_target = ''
if ts['statsd_prefix'] != '':
my_target = '$statsd_prefix'
if ts['ts_prefix'] != '':
my_target += '.$ts_prefix'
if ts.get('realms_prefix'):
my_target += '.' + Timeseries.get_realms_prefix(item['_realm'])
if 'host' in item:
my_target += '.' + item['hostname'] + '.' + item['name']
else:
my_target += '.' + item['name']
my_target += '.' + fields['name']
while my_target.startswith('.'):
my_target = my_target[1:]
# Sanitize field name for Graphite:
# + becomes a _
my_target = my_target.replace("+", "_")
# / becomes a -
my_target = my_target.replace("/", "-")
# space becomes a _
my_target = my_target.replace(" ", "_")
# % becomes _pct
my_target = my_target.replace("%", "_pct")
# all character not in [a-zA-Z_-0-9.] is removed
my_target = re.sub(r'[^a-zA-Z_\-0-9\.\$]', '', my_target)
# Build path for each metric
targets = {'main': {'name': fields['name'], 'target': my_target}}
overrides = []
if fields['warning'] is not None:
targets.update({
'warning': {
'name': fields['name'] + ' (w)',
'target': my_target + '_warning'
}
})
overrides.append({
'alias': fields['name'] + ' (w)',
'fill': 0,
'legend': False,
'color': '#CCA300'
})
if fields['critical'] is not None:
targets.update({
'critical': {
'name': fields['name'] + ' (c)',
'target': my_target + '_critical'
}
})
overrides.append({
'alias': fields['name'] + ' (c)',
'fill': 0,
'legend': False,
'color': '#890F02'
})
if fields['min'] is not None:
targets.update({
'min': {
'name': fields['name'] + ' (min)',
'target': my_target + '_min'
}
})
overrides.append({
'alias': fields['name'] + ' (min)',
'fill': 0,
'legend': False,
'color': '#447EBC'
})
if fields['max'] is not None:
targets.update({
'max': {
'name': fields['name'] + ' (max)',
'target': my_target + '_max'
}
})
overrides.append({
'alias': fields['name'] + ' (max)',
'fill': 0,
'legend': False,
'color': '#447EBC'
})
alias = True
if alias:
for t_name, t_value in iteritems(targets):
targets[t_name] = "alias(" + t_value[
'target'] + ", '%s'" % t_value['name'] + ")"
return targets, overrides