def history_system(request, alert_id):
alert = alerts_model.get_by_id(alert_id)
server = server_model.get_by_id(alert['server'])
page = request.GET.get('page', 1)
page = int(page)
skip = 0
if page > 1:
skip = 100 * (page - 1)
total = alerts_history_model.count_notifications(alert_id=alert['_id'])
on_page = 100
if total > on_page:
total_pages = total//on_page
else:
total_pages = 1
total_pages = range(total_pages)
notifications = alerts_history_model.get_notifications_list(alert_id=alert['_id'], limit=100, skip=skip)
return render(request, 'alerts/history.html', {
'notifications': notifications,
'alert': alert,
'server': server,
"total_pages": total_pages,
"page": page
})
def history_system(request, alert_id):
alert = alerts_model.get_by_id(alert_id)
server = server_model.get_by_id(alert['server'])
page = request.GET.get('page', 1)
page = int(page)
skip = 0
if page > 1:
skip = 100 * (page - 1)
total = alerts_history_model.count_notifications(alert_id=alert['_id'])
on_page = 100
if total > on_page:
total_pages = total/on_page
else:
total_pages = 1
total_pages = range(total_pages)
notifications = alerts_history_model.get_notifications_list(alert_id=alert['_id'], limit=100, skip=skip)
return render(request, 'alerts/history.html', {
'notifications': notifications,
'alert': alert,
'server': server,
"total_pages": total_pages,
"page": page
})
def test_sort_and_filter(self):
self._cleanup()
server_model.collection.insert({"name": "check_sort_and_filter_default"})
server = server_model.collection.find_one()
for i in range(0, 10):
data = [{
'command': "check_sort_and_filter_default.rb",
'exit_code': 1,
'output': 'CheckDisk WARNING: / 83.35% bytes usage (29 GiB/35 GiB)'
}]
health_checks_results_model.save(data=data, server=server)
result = health_checks_model.sort_and_filter()
assert len(result.all_checks) == 1
assert result.all_checks[0]['last_check']
assert result.all_checks[0]['last_check']['status'] == 'warning'
self._cleanup()
for i in range(0, 10):
server_id = server_model.collection.insert({"name": "{0}_server_check_sort_and_filter_by_host".format(i)})
server = server_model.get_by_id(server_id)
# exit_codes = {0: "ok", 1: "warning", 2: "critical"}
exit_code = 2 if i <= 5 else 2
exit_code = 1 if i > 5 else exit_code
for j in range(0, 100):
data = [{
'command': '{0}_check_sort_and_filter_by_host.rb'.format(i),
'exit_code': exit_code,
'output': 'CheckBanner OK: port 22 open'
}]
health_checks_results_model.save(data=data, server=server)
result = health_checks_model.sort_and_filter(sort_by='status')
assert len(result.sorted_result) == 10
for i in range(0, 10):
status = 'critical' if i <= 5 else 'ok'
status = 'warning' if i > 5 else status
assert result.sorted_result[i]['last_check']['status'] == status
result = health_checks_model.sort_and_filter(sort_by='host')
assert len(result.sorted_result) == 10
for i in range(0, 10):
assert result.sorted_result[i]['server']['name'] == "{0}_server_check_sort_and_filter_by_host".format(i)
result = health_checks_model.sort_and_filter(sort_by='host', filter_by='critical')
assert len(result.sorted_result) == 6
result = health_checks_model.sort_and_filter(sort_by='host', filter_by='warning')
assert len(result.sorted_result) == 4
def delete_server(request, server_id=None):
server = server_model.get_by_id(server_id)
if server:
alerts_model.delete_server_alerts(server['_id'])
server_model.delete(server['_id'])
return redirect(reverse('servers'))
def delete_plugin(request, plugin_id=None, server_id=None):
server = server_model.get_by_id(server_id)
plugin = plugin_model.get_by_id(plugin_id)
if server and plugin:
plugin_model.delete(plugin=plugin, server=server)
messages.add_message(request, messages.INFO, 'Plugin deleted')
return redirect(reverse('edit_server', kwargs={"server_id": server_id}))
def get_system_data_after(server_id=None,
timestamp=None,
check=None,
enddate=None,
timezone='UTC',
device_id=None):
server = server_model.get_by_id(server_id)
data = []
now = datetime.utcnow()
active_checks = [
'memory', 'loadavg', 'cpu', 'disk', 'network.inbound',
'network.outbound'
]
if check in active_checks and timestamp:
if check in ['network.inbound', 'network.outbound']:
key = 'i' if check == 'network.inbound' else 'o'
filtered_servers = [server]
data = system_model.get_global_device_data_after(
timestamp=timestamp,
enddate=enddate,
filtered_servers=filtered_servers,
key=key,
timezone=timezone,
check='network')
elif check == 'disk':
data = system_model.get_device_data_after(timestamp=timestamp,
enddate=enddate,
server=server,
timezone=timezone,
check=check,
device_id=device_id)
else:
data = system_model.get_data_after(timestamp=timestamp,
enddate=enddate,
server=server,
check=check,
timezone=timezone)
try:
now_local = dateformatcharts_local(datetime_to_unixtime(now),
tz=timezone)
except:
now_local = False
response = {
'data': data,
'last_update': datetime_to_unixtime(now),
'now_local': now_local,
'chart_type': 'line'
}
return response
def test_save(self):
self._cleanup()
server_id = server_model.collection.insert({"name": "server_check_sort_and_filter_by_host"})
server = server_model.get_by_id(server_id)
command = "testmehere"
for i in range(0, 10):
health_checks_model.save(command=command, server=server)
assert health_checks_model.collection.find().count() == 1
def save_system_occurence(self, alerts, server_id=None):
server = server_model.get_by_id(server_id)
# Format: {'cpu': [{'value': 2.6899999999999977, 'rule': '4f55da92925d75158d0001e0'}}]}
for key, values_list in alerts.items():
for value in values_list:
data = {}
save_to_db = False
alert_on = value.get('value', None)
trigger = value.get('trigger', False)
rule_id = value.get('rule', None)
time = value.get('time', None)
if time is None:
time = unix_utc_now()
alert_on = "{0:.2f}".format(float(alert_on))
alert = self.get_by_id(rule_id)
data = {
"value": float(alert_on),
"time": time,
"trigger": trigger
}
if key == 'disk':
volume = value.get('volume', None)
volume_data = volumes_model.get_by_name(server, volume)
if volume_data:
data['volume'] = volume_data.get('_id', None)
if key == 'network':
interface = value.get('interface', None)
interface_data = interfaces_model.get_by_name(
server, interface)
if interface_data:
data['interface'] = interface_data.get('_id', None)
server_id = self.mongo.get_object_id(server_id)
# Check for tagged global alerts
alert_server = alert.get('server')
if alert_server == 'all':
if self._server_tags_in_alert(server=server, alert=alert):
save_to_db = True
else:
save_to_db = True
if save_to_db:
self.alert_history_model.save(alert=alert,
server_id=server_id,
data=data)
def get_all(self):
result_list = []
result = super(AlertMuteServersModel, self).get_all()
for r in result:
tags = r.get('tags', [])
r['tags'] = [tags_model.get_by_id(x) for x in tags]
r['server'] = server_model.get_by_id(r.get('server'))
result_list.append(r)
return result_list
def test_save(self):
self._cleanup()
server_id = server_model.collection.insert(
{"name": "server_check_sort_and_filter_by_host"})
server = server_model.get_by_id(server_id)
command = "testmehere"
for i in range(0, 10):
health_checks_model.save(command=command, server=server)
assert health_checks_model.collection.find().count() == 1
def ajax_get_tags_for_server(request, server_id=None):
result = []
server = server_model.get_by_id(server_id)
server_tags = server_model.get_tags(server=server)
for tag in server_tags:
tag_dict = _tag_dict__repr__(tag=tag)
if len(tag_dict) > 0:
result.append(tag_dict)
result = sorted(result, key=itemgetter('group'))
return Response(result)
def get_all(self):
result_list = []
result = super(AlertMuteServersModel, self).get_all()
for r in result:
tags = r.get('tags', [])
r['tags'] = [tags_model.get_by_id(x) for x in tags]
r['server'] = server_model.get_by_id(r.get('server'))
result_list.append(r)
return result_list
def test_get_unique_commands(self):
self._cleanup()
server_id = server_model.collection.insert({"name": "server_check_sort_and_filter_by_host"})
server = server_model.get_by_id(server_id)
for i in range(0, 10):
command = "testcommand{0} -w {0} -c {0}".format(i)
health_checks_model.save(command=command, server=server)
result = health_checks_api_model.get_unique_commands()
assert len(result) == 10
def test_get_unique_commands(self):
self._cleanup()
server_id = server_model.collection.insert(
{"name": "server_check_sort_and_filter_by_host"})
server = server_model.get_by_id(server_id)
for i in range(0, 10):
command = "testcommand{0} -w {0} -c {0}".format(i)
health_checks_model.save(command=command, server=server)
result = health_checks_api_model.get_unique_commands()
assert len(result) == 10
def get_alerts(self, type=None, server=None, limit=None):
params = {"rule_type": type}
if server:
params['server'] = server['_id']
rules_list = []
rules = self.collection.find(params).count()
if rules > 0:
rules = self.collection.find(params)
rules_list = []
for rule in rules:
process_id = rule.get('process', None)
if process_id:
rule['process_data'] = process_model.get_by_id(process_id)
plugin_id = rule.get('plugin', None)
gauge_id = rule.get('gauge', None)
if plugin_id and gauge_id:
rule['plugin_data'] = plugin_model.get_by_id(plugin_id)
rule['gauge_data'] = plugin_model.get_gauge_by_id(gauge_id)
if server:
rule['server'] = server
# Check if the rule is for specific server and get the data
else:
rule_server = rule.get('server', False)
server_id = self.object_id(rule_server)
if server_id:
rule['server'] = server_model.get_by_id(rule_server)
tags = rule.get('tags', False)
if tags:
rule = self._get_alert_tags(rule)
rule['notifications'] = self._get_notifications(rule)
rule[
'last_trigger'] = self.alert_history_model.get_last_trigger(
alert_id=rule['_id'])
rule[
'total_triggers'] = self.alert_history_model.count_notifications(
alert_id=rule['_id'])
rules_list.append(rule)
return rules_list
def get_notifications_list(self, alert_id=None, server=None, limit=0, skip=0):
notifications_list = []
server_id = None
if server:
server_id = server.get('_id') # Could be none
notifications = self.get_notifications(alert_id=alert_id, server_id=server_id, limit=limit, skip=skip)
if notifications['count'] > 0:
for notification in notifications['data']:
notification_dict = {
"period_from": notification['from'],
"period_to": notification['time'],
"average_value": notification['average_value'],
"id": notification.get('_id')
}
# System notifications specific here
server = server_model.get_by_id(notification.get('server_id'))
if server:
notification_dict['server_id'] = server['_id']
notification_dict['server'] = server['name']
notification_dict['last_check'] = server.get('last_check')
volume = notification.get('volume')
if volume:
notification_dict['volume_data'] = volumes_model.get_by_id(volume)
interface = notification.get('interface')
if interface:
notification_dict['interface_data'] = interfaces_model.get_by_id(interface)
health_checks_data_id = notification.get("health_checks_data_id")
if health_checks_data_id:
health_check_data = health_checks_results_model.get_by_id(health_checks_data_id)
if type(health_check_data) is dict:
health_check_id = health_check_data.get('check_id')
notification_dict['health_check'] = health_checks_model.get_by_id(health_check_id)
notification_dict['health_check_data'] = health_check_data
notifications_list.append(notification_dict)
return notifications_list
def test_get_commands_for_server(self):
self._cleanup()
server_id = server_model.collection.insert({"name": "server_check_sort_and_filter_by_host"})
server = server_model.get_by_id(server_id)
command = "testmehere -w 10"
for i in range(0, 10):
health_checks_model.save(command=command, server=server)
second_command = "testmeagain -c 10"
for i in range(0, 5):
health_checks_model.save(command=second_command, server=server)
result = health_checks_api_model.get_commands_for_server(server_id=server['_id'])
assert result.count() == 2
def test_get_params_for_command(self):
self._cleanup()
server_id = server_model.collection.insert({"name": "server_check_sort_and_filter_by_host"})
server = server_model.get_by_id(server_id)
for i in range(0, 10):
command = "testcommand -w {0} -c {0}".format(i)
health_checks_model.save(command=command, server=server)
# Duplicate - still has to return only 10 unique params
for i in range(0, 10):
command = "testcommand -w {0} -c {0}".format(i)
health_checks_model.save(command=command, server=server)
result = health_checks_api_model.get_params_for_command(command_string="testcommand")
assert len(result) == 10
def get_alerts_not_sending_data(self, metric=None):
params = {"metric": 'NotSendingData'}
alerts_list = []
result = self.collection.find(params)
if result.clone().count() > 0:
for rule in result:
rule['notifications'] = self._get_notifications(rule)
server = rule.get('server')
if server:
rule['server_data'] = [server_model.get_by_id(server)]
if rule['rule_type'] == 'global':
rule['server_data'] = server_model.get_all()
alerts_list.append(rule)
return alerts_list
def test_get_commands_for_server(self):
self._cleanup()
server_id = server_model.collection.insert(
{"name": "server_check_sort_and_filter_by_host"})
server = server_model.get_by_id(server_id)
command = "testmehere -w 10"
for i in range(0, 10):
health_checks_model.save(command=command, server=server)
second_command = "testmeagain -c 10"
for i in range(0, 5):
health_checks_model.save(command=second_command, server=server)
result = health_checks_api_model.get_commands_for_server(
server_id=server['_id'])
assert result.count() == 2
def _fill_metrics_arrays(all_metrics=None):
charts_list = []
health_checks_list = []
for m in all_metrics:
metric_type = m.get('type')
if metric_type == 'healthcheck':
check_id = m.get('healthcheck_id')
server = None
if check_id:
check = health_checks_model.get_by_id(check_id)
server = server_model.get_by_id(check.get('server_id', None))
if server != None:
check['server'] = server
health_checks_list.append(check)
else:
charts_list.append(m)
return {'charts': charts_list, 'health_checks': health_checks_list}
def get_system_data_after(server_id=None, timestamp=None, check=None, enddate=None, timezone='UTC', device_id=None):
server = server_model.get_by_id(server_id)
data = []
now = datetime.utcnow()
active_checks = ['memory', 'loadavg', 'cpu', 'disk', 'network.inbound', 'network.outbound']
if check in active_checks and timestamp:
if check in ['network.inbound', 'network.outbound']:
key = 'i' if check == 'network.inbound' else 'o'
filtered_servers = [server]
data = system_model.get_global_device_data_after(timestamp=timestamp,
enddate=enddate,
filtered_servers=filtered_servers,
key=key,
timezone=timezone,
check='network'
)
elif check == 'disk':
data = system_model.get_device_data_after(timestamp=timestamp, enddate=enddate, server=server, timezone=timezone,
check=check, device_id=device_id)
else:
data = system_model.get_data_after(timestamp=timestamp, enddate=enddate, server=server, check=check, timezone=timezone)
try:
now_local = dateformatcharts_local(datetime_to_unixtime(now), tz=timezone)
except:
now_local = False
response = {
'data': data,
'last_update': datetime_to_unixtime(now),
'now_local': now_local,
'chart_type': 'line'
}
return response
def test_get_params_for_command(self):
self._cleanup()
server_id = server_model.collection.insert(
{"name": "server_check_sort_and_filter_by_host"})
server = server_model.get_by_id(server_id)
for i in range(0, 10):
command = "testcommand -w {0} -c {0}".format(i)
health_checks_model.save(command=command, server=server)
# Duplicate - still has to return only 10 unique params
for i in range(0, 10):
command = "testcommand -w {0} -c {0}".format(i)
health_checks_model.save(command=command, server=server)
result = health_checks_api_model.get_params_for_command(
command_string="testcommand")
assert len(result) == 10
def test_delete(self):
self._cleanup()
server_id = server_model.collection.insert({"name": "server_check_sort_and_filter_by_host"})
server = server_model.get_by_id(server_id)
command = "testmehere"
for i in range(0, 5):
health_checks_model.save(command=command, server=server)
result = health_checks_model.collection.count()
check = health_checks_model.collection.find_one()
assert result == 1
health_checks_model.delete(check_id=check['_id'])
result = health_checks_model.collection.count()
assert result == 0
def edit_server(request, server_id=None):
server = server_model.get_by_id(server_id)
plugins = plugin_model.get_for_server(server_id)
form = ServerForm(server=server)
if request.method == 'POST':
form = ServerForm(request.POST, server=server)
if form.is_valid():
form.save()
messages.add_message(request, messages.INFO, 'Server settings updated.')
return redirect(reverse('servers'))
return render_to_response('servers/edit.html', {
"server": server,
"plugins": plugins,
'form': form,
}, context_instance=RequestContext(request))
def test_delete(self):
self._cleanup()
server_id = server_model.collection.insert(
{"name": "server_check_sort_and_filter_by_host"})
server = server_model.get_by_id(server_id)
command = "testmehere"
for i in range(0, 5):
health_checks_model.save(command=command, server=server)
result = health_checks_model.collection.count()
check = health_checks_model.collection.find_one()
assert result == 1
health_checks_model.delete(check_id=check['_id'])
result = health_checks_model.collection.count()
assert result == 0
def _fill_metrics_arrays(all_metrics=None):
charts_list = []
health_checks_list = []
for m in all_metrics:
metric_type = m.get('type')
if metric_type == 'healthcheck':
check_id = m.get('healthcheck_id')
server = None
if check_id:
check = health_checks_model.get_by_id(check_id)
server = server_model.get_by_id(check.get('server_id', None))
if server != None:
check['server'] = server
health_checks_list.append(check)
else:
charts_list.append(m)
return {
'charts': charts_list,
'health_checks': health_checks_list
}
def generate_notifications():
notifications_list = []
unsent_alerts = alerts_history_model.get_unsent()
for trigger in unsent_alerts.get('data'):
result = AmonStruct()
result.global_mute = False
metadata = None
timezone = 'UTC'
try:
alert = alerts_model.get_by_id(trigger['alert_id'])
except:
alert = None # Deleted alert here
if alert:
rule_type = alert.get('rule_type', 'system')
metric_type = alert.get('metric', None)
else:
rule_type = 'alert-does-not-exist'
if rule_type in ['global', 'process_global', 'plugin_global', 'process', 'system', 'plugin', 'uptime', 'health_check']:
if rule_type in ['global', 'process_global', 'plugin_global', 'health_check']:
server_id = trigger.get('server_id')
else:
server_id = alert.get('server')
if server_id:
server = server_model.get_by_id(server_id)
result.server = server
result.global_mute = alert_mute_servers_model.check_if_server_is_muted(server=server)
if metric_type:
metric_type = metric_type.lower()
if metric_type in ['cpu', 'memory', 'loadavg']:
trigger_period_from = trigger['from']
trigger_period_to = trigger['time']
metric_type = 'cpu' if metric_type == 'loadavg' else metric_type # Get CPU top consumers for Load average
if server:
metadata = process_model.get_top_consumers_for_period(date_from=trigger_period_from,
date_to=trigger_period_to, server=server, metric_type=metric_type)
# Overwrite rule_type for the new type
if metric_type == 'notsendingdata':
alert['rule_type'] = 'notsendingdata'
if metric_type == 'disk':
volume_id = trigger.get('volume')
metadata = volumes_model.get_by_id(volume_id)
if metric_type in ['network/inbound', 'network/outbound']:
interface_id = trigger.get('interface')
metadata = interfaces_model.get_by_id(interface_id)
if rule_type == 'process_global':
process_name = alert.get('process')
result.process = process_model.get_by_name_and_server_id(server_id=server_id, name=process_name)
if rule_type == 'plugin_global':
gauge_name = alert.get('gauge')
plugin_name = alert.get('plugin')
result.plugin = plugin_model.get_by_name_and_server_id(server_id=server_id, name=plugin_name)
result.gauge = plugin_model.get_gauge_by_name_and_plugin_id(plugin=result.plugin, name=gauge_name)
# Process and Uptime alerts
if rule_type == 'process' or rule_type == 'uptime':
process_dict = alert.get('process')
if process_dict:
result.process = process_model.get_by_id(process_dict.get('_id'))
if rule_type == 'plugin':
result.plugin = alert.get('plugin')
result.gauge = alert.get('gauge')
if rule_type == 'health_check':
health_check_result_id = trigger.get('health_checks_data_id')
health_check_result = health_checks_results_model.get_by_id(health_check_result_id)
if type(health_check_result) is dict:
health_check_id = health_check_result.get('check_id')
health_check = health_checks_model.get_by_id(health_check_id)
result.healthcheck = health_check
result.health_check_result = health_check_result
if alert:
result.alert = alert
result.metadata = metadata
result.timezone = timezone
result.trigger = trigger
result.mute = alert.get('mute', False) # Shortcut
notifications_list.append(result)
return notifications_list
def sort_and_filter(self, sort_by=None, filter_by=None):
flat_list = []
sorted_result = []
all_checks = []
all_servers = server_model.get_all()
for check in self.get_all():
last_check = check.get('last_check')
try:
server = server_model.get_by_id(check['server_id'])
except:
server = None
check['server'] = server
# Append result only for existing servers
if server != None:
flat_list.append(check)
if sort_by is None and filter_by is None:
if server != None:
all_checks.append(check)
sort_priority = {'critical': 1, 'warning': 2, 'ok': 3, 'unknown': 4}
count_statuses = {'critical': 0, 'warning': 0, 'ok': 0, 'unknown': 0}
for r in flat_list:
result = r.get("last_check", {})
if result:
check_status = result.get('status')
try:
count_statuses[check_status] = count_statuses[check_status] + 1
except:
pass
if filter_by:
reodered_list = []
for el in flat_list:
check_status = el.get('last_check', {}).get('status')
if check_status == filter_by:
reodered_list.append(el)
sorted_result = sorted(reodered_list, key=lambda d: d.get('last_check', {}).get('status'))
if sort_by:
# ?sort_by=host&filter_by=critical
if filter_by:
flat_list = sorted_result
if sort_by == 'status':
reodered_list = []
for el in flat_list:
try:
el['priority'] = sort_priority.get(el.get('last_check', {}).get('status'))
except:
pass
reodered_list.append(el)
sorted_result = sorted(reodered_list, key=itemgetter('priority'))
elif sort_by == 'host':
sorted_result = sorted(flat_list, key=lambda d: d.get('server', {}).get('name'))
result = AmonStruct()
result.all_checks = all_checks
result.sorted_result = sorted_result
result.count_statuses = count_statuses
result.flat_list = flat_list
return result
def system_view(request, server_id):
enddate = request.GET.get('enddate')
charts = request.GET.get('charts', 'all')
duration = request.GET.get('duration', 10800)
duration = int(duration)
server = server_model.get_by_id(server_id)
tags = server_model.get_tags(server=server)
if tags:
server['tags'] = tags
first_check_date = system_model.get_first_check_date(server)
now_unix = datetime_to_unixtime(request.now)
max_date = now_unix * 1000
if enddate:
date_to = int(enddate)
else:
date_to = now_unix
date_from = date_to - int(duration)
data_url = reverse('ajax_get_data_after')
data_url = "{0}?server_id={1}".format(data_url, server.get('_id'))
charts_metadata = {
'cpu': {
'id': 'cpu',
'check': 'cpu',
'name': 'CPU',
'url': data_url,
},
'loadavg': {
'id': 'loadavg',
'check': 'loadavg',
'name': 'Load Average',
'url': data_url
},
'memory': {
'id': 'memory',
'check': 'memory',
'name': 'Memory',
'url': data_url,
'type': 'area'
},
'network': [{
'id': 'network.inbound',
'check': 'network.inbound',
'name': 'Network - KB/s Received',
'url': data_url,
'unit': 'kb/s'
}, {
'id': 'network.outbound',
'check': 'network.outbound',
'name': 'Network - KB/s Sent',
'url': data_url,
'unit': 'kb/s'
}]
}
if charts == 'all':
active_checks = ['cpu', 'memory', 'loadavg', 'disk', 'network']
else:
active_checks = [charts]
selected_charts = []
for check in active_checks:
if check in ['network']:
chart_meta = charts_metadata.get(check)
for i in chart_meta:
selected_charts.append(i)
elif check != 'disk':
chart_meta = charts_metadata.get(check)
selected_charts.append(chart_meta)
volumes = volumes_model.get_all_for_server(server_id)
if 'disk' in active_checks:
unit = get_disk_unit(server)
for device in volumes.clone():
device_id, name = device.get('_id'), device.get('name')
url = "{0}&device_id={1}".format(data_url, device_id)
meta = {
'id': device_id,
'check': 'disk',
'name': name,
'url': url,
'unit': unit
}
last_update = device.get('last_update')
if last_update > date_from:
selected_charts.append(meta)
all_plugins = plugin_model.get_for_server(server_id=server['_id'],
last_check_after=date_from)
all_processes = process_model.get_all_for_server(
server_id, last_check_after=date_from)
breadcrumb_url = reverse('server_system',
kwargs={'server_id': server['_id']})
breadcrumb_url = "{0}?charts={1}".format(breadcrumb_url, charts)
return render(
request, 'system/view.html', {
"enddate": enddate,
"duration": duration,
"all_processes": all_processes,
"all_plugins": all_plugins,
"now": now_unix,
"charts": charts,
"selected_charts": selected_charts,
"date_from": date_from,
"date_to": date_to,
"first_check_date": first_check_date,
"server": server,
"max_date": max_date,
"server_id": server_id,
"breadcrumb_url": breadcrumb_url
})
def save_data_test(self):
self._cleanup()
expires_at = datetime.utcnow() + timedelta(hours=24)
last_check = 99999
system_data = {
u'disk': {
u'sda1': {
u'used': u'21350',
u'percent': u'46',
u'free': u'25237',
u'volume': u'/dev/sda1',
u'path': u'/',
u'total': u'49086'
}
},
u'memory': {
u'used_percent': 34,
u'swap_used_mb': 0,
u'total_mb': 3954,
u'free_mb': 2571,
u'swap_used_percent': 0,
u'swap_free_mb': 0,
u'used_mb': 1383,
u'swap_total_mb': 0
},
u'loadavg': {
u'cores': 4,
u'fifteen_minutes': u'0.36',
u'minute': u'0.12',
u'five_minutes': u'0.31'
},
u'network': {
u'eth3': {
u'inbound': u'6.05',
u'outbound': u'1.97'
}
},
u'cpu': {
u'iowait': u'0.00',
u'system': u'1.32',
u'idle': u'98.68',
u'user': u'0.00',
u'steal': u'0.00',
u'nice': u'0.00'
}
}
system_model.save_data(self.server,
system_data,
time=last_check,
expires_at=expires_at)
disk_collection = volumes_model.get_data_collection(
server_id=self.server['_id'])
disk = volumes_model.get_by_name(server=self.server, name='sda1')
eq_(disk_collection.find().count(), 1)
for r in disk_collection.find():
eq_(r['t'], last_check)
eq_(r['total'], "49086")
eq_(r['used'], "21350")
eq_(r['device_id'], disk['_id'])
eq_(r['expires_at'].date(), expires_at.date())
network_collection = interfaces_model.get_data_collection(
server_id=self.server['_id'])
adapter = interfaces_model.get_by_name(server=self.server, name='eth3')
eq_(network_collection.find().count(), 1)
for r in network_collection.find():
eq_(r['t'], last_check)
eq_(r['i'], "6.05")
eq_(r['o'], "1.97")
eq_(r['device_id'], adapter['_id'])
eq_(r['expires_at'].date(), expires_at.date())
data_collection = system_model.data_collection
for r in data_collection.find():
eq_(r['time'], last_check)
eq_(r['memory']['free_mb'], 2571)
eq_(r['loadavg']['fifteen_minutes'], '0.36')
eq_(r['cpu']['system'], '1.32')
eq_(r['expires_at'].date(), expires_at.date())
server_updated = server_model.get_by_id(self.server['_id'])
eq_(server_updated['last_check'], last_check)
self._cleanup()
def get_all(self, account_id=None, dashboard_id=None, public=None):
result_list = []
query = []
params = {'dashboard_id': dashboard_id}
params = self.keys_to_mongoid(data=params, keys=['dashboard_id'])
if dashboard_id:
query = super(DashboardMetricsModel, self).get(params=params)
utc_now = unix_utc_now()
for metric in query:
mongo_id = metric.get('_id')
server_id = metric.get('server_id')
metric_type = metric.get('metric_type')
unique_id = metric.get('unique_id')
check = metric.get('check')
order = metric.get('order', 0)
tags = metric.get('tags', [])
tags_list = tags_model.get_list_of_tags(tags_list=tags, to_dict=True)
server = server_model.get_by_id(server_id)
process = process_model.get_by_id(metric.get('process_id'))
plugin = plugin_model.get_by_id(metric.get('plugin_id'))
gauge = plugin_model.get_gauge_by_id(gauge_id=metric.get('gauge_id'))
volume = volumes_model.get_by_id(metric.get('device_id'))
interface = interfaces_model.get_by_id(metric.get('device_id'))
healthcheck_metric = health_checks_model.get_by_id(metric.get('healthcheck_id'))
append = False
unit = yaxis(check)
if metric_type == 'system_global' and check == 'memory':
unit = "%"
if metric_type == 'system_global' and check == 'disk':
unit = '%'
if public:
url = reverse('public_dashboard_metric', kwargs={"metric_id": mongo_id})
else:
url = reverse('dashboard_metric', kwargs={"metric_id": mongo_id})
result = {
'id': mongo_id,
'unique_id': unique_id,
'metric_type': metric_type,
'url': url,
'utcnow': utc_now,
'name': '',
'unit': unit,
'tags': tags_list,
'order': order
}
if server:
result.update({'server_id': server_id, 'type': 'server_metric','server_name' :server.get('name')})
if metric_type == 'system':
result['name'] = "{0}".format(check)
if volume:
result['name'] = u"{0}.{1}".format(result['name'], volume['name'])
if interface:
result['name'] = u"{0}.{1}".format(result['name'], interface['name'])
append = True
elif metric_type == 'process' and process:
process_name = process.get('name')
result['name'] = u"{0}.{1}".format(process_name, check)
result['process_id'] = process['_id']
append = True
elif metric_type == 'plugin' and plugin and gauge:
result['name'] = u"{0}.{1}".format(plugin.get('name'), gauge.get('name'))
result['plugin_id'] = plugin['_id']
result['gauge_id'] = gauge['_id']
append = True
result['name'] = u"{0}.{1}".format(server.get('name'), result['name'])
elif healthcheck_metric:
result['healthcheck'] = healthcheck_metric
result['healthcheck_id'] = healthcheck_metric.get('_id')
try:
del result['healthcheck']['_id']
del result['healthcheck']['server_id']
del result['healthcheck']['tags']
del result['healthcheck']['file_id'] # Custom scripts
except:
pass
result['type'] = 'healthcheck'
append = True
else:
key = metric.get('key')
# Overwrite keys for better visual presentation
if check == 'network':
key = 'inbound' if key == 'i' else 'outbound'
result['name'] = u"{0}.{1}".format(check, key)
append = True
if metric_type == 'plugin_global':
result['name'] = u'{0}.{1}.{2}'.format(metric.get('plugin'), metric.get('gauge'), metric.get('key'))
append = True
result = self.mongoid_to_str(result, ['server_id', 'id', 'process_id', 'plugin_id', 'metric_id', 'gauge_id', 'healthcheck_id',])
if append:
result_list.append(result)
from operator import itemgetter
sorted_list = sorted(result_list, key=itemgetter('order'))
return sorted_list
def get_all(self, account_id=None, dashboard_id=None, public=None):
result_list = []
query = []
params = {'dashboard_id': dashboard_id}
params = self.keys_to_mongoid(data=params, keys=['dashboard_id'])
if dashboard_id:
query = super(DashboardMetricsModel, self).get(params=params)
utc_now = unix_utc_now()
for metric in query:
mongo_id = metric.get('_id')
server_id = metric.get('server_id')
metric_type = metric.get('metric_type')
unique_id = metric.get('unique_id')
check = metric.get('check')
order = metric.get('order', 0)
tags = metric.get('tags', [])
tags_list = tags_model.get_list_of_tags(tags_list=tags,
to_dict=True)
server = server_model.get_by_id(server_id)
process = process_model.get_by_id(metric.get('process_id'))
plugin = plugin_model.get_by_id(metric.get('plugin_id'))
gauge = plugin_model.get_gauge_by_id(
gauge_id=metric.get('gauge_id'))
volume = volumes_model.get_by_id(metric.get('device_id'))
interface = interfaces_model.get_by_id(metric.get('device_id'))
healthcheck_metric = health_checks_model.get_by_id(
metric.get('healthcheck_id'))
append = False
unit = yaxis(check)
if metric_type == 'system_global' and check == 'memory':
unit = "%"
if metric_type == 'system_global' and check == 'disk':
unit = '%'
if public:
url = reverse('public_dashboard_metric',
kwargs={"metric_id": mongo_id})
else:
url = reverse('dashboard_metric',
kwargs={"metric_id": mongo_id})
result = {
'id': mongo_id,
'unique_id': unique_id,
'metric_type': metric_type,
'url': url,
'utcnow': utc_now,
'name': '',
'unit': unit,
'tags': tags_list,
'order': order
}
if server:
result.update({
'server_id': server_id,
'type': 'server_metric',
'server_name': server.get('name')
})
if metric_type == 'system':
result['name'] = "{0}".format(check)
if volume:
result['name'] = u"{0}.{1}".format(
result['name'], volume['name'])
if interface:
result['name'] = u"{0}.{1}".format(
result['name'], interface['name'])
append = True
elif metric_type == 'process' and process:
process_name = process.get('name')
result['name'] = u"{0}.{1}".format(process_name, check)
result['process_id'] = process['_id']
append = True
elif metric_type == 'plugin' and plugin and gauge:
result['name'] = u"{0}.{1}".format(plugin.get('name'),
gauge.get('name'))
result['plugin_id'] = plugin['_id']
result['gauge_id'] = gauge['_id']
append = True
result['name'] = u"{0}.{1}".format(server.get('name'),
result['name'])
elif healthcheck_metric:
result['healthcheck'] = healthcheck_metric
result['healthcheck_id'] = healthcheck_metric.get('_id')
try:
del result['healthcheck']['_id']
del result['healthcheck']['server_id']
del result['healthcheck']['tags']
del result['healthcheck']['file_id'] # Custom scripts
except:
pass
result['type'] = 'healthcheck'
append = True
else:
key = metric.get('key')
# Overwrite keys for better visual presentation
if check == 'network':
key = 'inbound' if key == 'i' else 'outbound'
result['name'] = u"{0}.{1}".format(check, key)
append = True
if metric_type == 'plugin_global':
result['name'] = u'{0}.{1}.{2}'.format(metric.get('plugin'),
metric.get('gauge'),
metric.get('key'))
append = True
result = self.mongoid_to_str(result, [
'server_id',
'id',
'process_id',
'plugin_id',
'metric_id',
'gauge_id',
'healthcheck_id',
])
if append:
result_list.append(result)
from operator import itemgetter
sorted_list = sorted(result_list, key=itemgetter('order'))
return sorted_list
def system_view(request, server_id):
enddate = request.GET.get('enddate')
charts = request.GET.get('charts', 'all')
duration = request.GET.get('duration', 10800)
duration = int(duration)
server = server_model.get_by_id(server_id)
tags = server_model.get_tags(server=server)
if tags:
server['tags'] = tags
first_check_date = system_model.get_first_check_date(server)
now_unix = datetime_to_unixtime(request.now)
max_date = now_unix * 1000
if enddate:
date_to = int(enddate)
else:
date_to = now_unix
date_from = date_to - int(duration)
data_url = reverse('ajax_get_data_after')
data_url = "{0}?server_id={1}".format(data_url, server.get('_id'))
charts_metadata = {
'cpu': {'id': 'cpu', 'check': 'cpu', 'name': 'CPU', 'url': data_url, },
'loadavg': {'id': 'loadavg', 'check': 'loadavg', 'name': 'Load Average','url': data_url},
'memory': {'id': 'memory', 'check': 'memory', 'name': 'Memory', 'url': data_url, 'type': 'area'},
'network': [
{'id': 'network.inbound', 'check': 'network.inbound', 'name': 'Network - KB/s Received', 'url': data_url, 'unit': 'kb/s'},
{'id': 'network.outbound', 'check': 'network.outbound', 'name': 'Network - KB/s Sent', 'url': data_url, 'unit': 'kb/s'}
]
}
if charts == 'all':
active_checks = ['cpu', 'memory', 'loadavg', 'disk', 'network']
else:
active_checks = [charts]
selected_charts = []
for check in active_checks:
if check in ['network']:
chart_meta = charts_metadata.get(check)
for i in chart_meta:
selected_charts.append(i)
elif check != 'disk':
chart_meta = charts_metadata.get(check)
selected_charts.append(chart_meta)
volumes = volumes_model.get_all_for_server(server_id)
if 'disk' in active_checks:
unit = get_disk_unit(server)
for device in volumes.clone():
device_id, name = device.get('_id'), device.get('name')
url = "{0}&device_id={1}".format(data_url, device_id)
meta = {'id': device_id, 'check': 'disk', 'name': name, 'url': url , 'unit': unit}
last_update = device.get('last_update')
if last_update > date_from:
selected_charts.append(meta)
all_plugins = plugin_model.get_for_server(server_id=server['_id'], last_check_after=date_from)
all_processes = process_model.get_all_for_server(server_id, last_check_after=date_from)
breadcrumb_url = reverse('server_system', kwargs={'server_id': server['_id']})
breadcrumb_url = "{0}?charts={1}".format(breadcrumb_url, charts)
return render(request, 'system/view.html', {
"enddate": enddate,
"duration": duration,
"all_processes": all_processes,
"all_plugins": all_plugins,
"now": now_unix,
"charts":charts,
"selected_charts" : selected_charts,
"date_from" : date_from,
"date_to" : date_to,
"first_check_date" : first_check_date,
"server" : server,
"max_date" : max_date,
"server_id": server_id,
"breadcrumb_url": breadcrumb_url
})
def sort_and_filter(self, sort_by=None, filter_by=None):
flat_list = []
sorted_result = []
all_checks = []
all_servers = server_model.get_all()
for check in self.get_all():
last_check = check.get('last_check')
try:
server = server_model.get_by_id(check['server_id'])
except:
server = None
check['server'] = server
# Append result only for existing servers
if server != None:
flat_list.append(check)
if sort_by is None and filter_by is None:
if server != None:
all_checks.append(check)
sort_priority = {'critical': 1, 'warning': 2, 'ok': 3, 'unknown': 4}
count_statuses = {'critical': 0, 'warning': 0, 'ok': 0, 'unknown': 0}
for r in flat_list:
result = r.get("last_check", {})
if result:
check_status = result.get('status')
try:
count_statuses[
check_status] = count_statuses[check_status] + 1
except:
pass
if filter_by:
reodered_list = []
for el in flat_list:
check_status = el.get('last_check', {}).get('status')
if check_status == filter_by:
reodered_list.append(el)
sorted_result = sorted(
reodered_list,
key=lambda d: d.get('last_check', {}).get('status'))
if sort_by:
# ?sort_by=host&filter_by=critical
if filter_by:
flat_list = sorted_result
if sort_by == 'status':
reodered_list = []
for el in flat_list:
try:
el['priority'] = sort_priority.get(
el.get('last_check', {}).get('status'))
except:
pass
reodered_list.append(el)
sorted_result = sorted(reodered_list,
key=itemgetter('priority'))
elif sort_by == 'host':
sorted_result = sorted(
flat_list, key=lambda d: d.get('server', {}).get('name'))
result = AmonStruct()
result.all_checks = all_checks
result.sorted_result = sorted_result
result.count_statuses = count_statuses
result.flat_list = flat_list
return result
def test_sort_and_filter(self):
self._cleanup()
server_model.collection.insert(
{"name": "check_sort_and_filter_default"})
server = server_model.collection.find_one()
for i in range(0, 10):
data = [{
'command':
"check_sort_and_filter_default.rb",
'exit_code':
1,
'output':
'CheckDisk WARNING: / 83.35% bytes usage (29 GiB/35 GiB)'
}]
health_checks_results_model.save(data=data, server=server)
result = health_checks_model.sort_and_filter()
assert len(result.all_checks) == 1
assert result.all_checks[0]['last_check']
assert result.all_checks[0]['last_check']['status'] == 'warning'
self._cleanup()
for i in range(0, 10):
server_id = server_model.collection.insert(
{"name": "{0}_server_check_sort_and_filter_by_host".format(i)})
server = server_model.get_by_id(server_id)
# exit_codes = {0: "ok", 1: "warning", 2: "critical"}
exit_code = 2 if i <= 5 else 2
exit_code = 1 if i > 5 else exit_code
for j in range(0, 100):
data = [{
'command':
'{0}_check_sort_and_filter_by_host.rb'.format(i),
'exit_code':
exit_code,
'output':
'CheckBanner OK: port 22 open'
}]
health_checks_results_model.save(data=data, server=server)
result = health_checks_model.sort_and_filter(sort_by='status')
assert len(result.sorted_result) == 10
for i in range(0, 10):
status = 'critical' if i <= 5 else 'ok'
status = 'warning' if i > 5 else status
assert result.sorted_result[i]['last_check']['status'] == status
result = health_checks_model.sort_and_filter(sort_by='host')
assert len(result.sorted_result) == 10
for i in range(0, 10):
assert result.sorted_result[i]['server'][
'name'] == "{0}_server_check_sort_and_filter_by_host".format(i)
result = health_checks_model.sort_and_filter(sort_by='host',
filter_by='critical')
assert len(result.sorted_result) == 6
result = health_checks_model.sort_and_filter(sort_by='host',
filter_by='warning')
assert len(result.sorted_result) == 4
