def get(self):
processes = self.get_arguments('processes', None)
date_from = self.get_argument('date_from', False)
date_to = self.get_argument('date_to', False)
if date_from:
date_from = datestring_to_utc_datetime(date_from)
# Default - 24 hours period
else:
day = timedelta(hours=24)
date_from = self.now - day
if date_to:
date_to = datestring_to_utc_datetime(date_to)
else:
date_to = self.now
date_from = datetime_to_unixtime(date_from)
date_to = datetime_to_unixtime(date_to)
all_processes_checks = settings.PROCESS_CHECKS
if len(processes) > 0:
processes_checks = processes
else:
processes_checks = settings.PROCESS_CHECKS
process_data = process_model.get_process_data(processes_checks, date_from, date_to)
# Convert the dates to local time for display
date_from = utc_unixtime_to_localtime(date_from)
date_to = utc_unixtime_to_localtime(date_to)
# Get the difference between UTC and localtime - used to display
# the ticks in the charts
zone_difference = localtime_utc_timedelta()
# Get the max date - utc, converted to localtime
max_date = utc_now_to_localtime()
self.render('processes.html',
current_page=self.current_page,
all_processes_checks=all_processes_checks,
processes_checks=processes_checks,
processes=processes,
process_data=process_data,
date_from=date_from,
date_to=date_to,
zone_difference=zone_difference,
max_date=max_date
)
def get(self):
processes = self.get_arguments('processes', None)
date_from = self.get_argument('date_from', False)
date_to = self.get_argument('date_to', False)
if date_from:
date_from = datestring_to_utc_datetime(date_from)
# Default - 24 hours period
else:
day = timedelta(hours=24)
date_from = self.now - day
if date_to:
date_to = datestring_to_utc_datetime(date_to)
else:
date_to = self.now
date_from = datetime_to_unixtime(date_from)
date_to = datetime_to_unixtime(date_to)
all_processes_checks = settings.PROCESS_CHECKS
if len(processes) > 0:
processes_checks = processes
else:
processes_checks = settings.PROCESS_CHECKS
process_data = process_model.get_process_data(processes_checks,
date_from, date_to)
# Convert the dates to local time for display
date_from = utc_unixtime_to_localtime(date_from)
date_to = utc_unixtime_to_localtime(date_to)
# Get the difference between UTC and localtime - used to display
# the ticks in the charts
zone_difference = localtime_utc_timedelta()
# Get the max date - utc, converted to localtime
max_date = utc_now_to_localtime()
self.render('processes.html',
current_page=self.current_page,
all_processes_checks=all_processes_checks,
processes_checks=processes_checks,
processes=processes,
process_data=process_data,
date_from=date_from,
date_to=date_to,
zone_difference=zone_difference,
max_date=max_date)
def generate_charts(self,
result=None,
keys=None,
check=None,
timezone='UTC'):
keys_length = len(keys)
data_lists = []
data_lists = [[] for i in range(keys_length)]
for r in result:
time = r.get('time') if r.get('time') else r.get('t')
time = utc_unixtime_to_localtime(time, tz=timezone)
for i in range(keys_length):
data = r.get(check) if r.get(check) else r
key = keys[i].key
value = data.get(
key) # Gets to the specific key -> cpu: {'steal': 4}
value = self.format_float(value)
data_lists[i].append({"x": time, "y": value})
series = []
for i in range(keys_length):
data = [] if len(data_lists[i]) == 0 else data_lists[i]
chart_colors = select_colors(i)
metric_type = {'name': keys[i].name, 'data': data, 'unit': '%'}
metric_type.update(chart_colors)
series.append(metric_type)
return series
def get_global_data_after(self,
timestamp=None,
key=None,
enddate=None,
check=None,
timezone='UTC',
filtered_servers=None):
params = {"time": {"$gte": int(timestamp)}}
if enddate:
params['time']["$lte"] = int(enddate)
keys = {}
data_lists = {}
for server in filtered_servers:
server_id = str(server['_id'])
keys[server_id] = server.get('name')
data_lists[server_id] = []
server_ids_list = [x.get('_id') for x in filtered_servers]
params['server_id'] = {"$in": server_ids_list}
result = self.data_collection.find(params).sort('time', self.asc)
for r in result:
r_server_id = str(r['server_id'])
time = r.get('time', 0)
time = utc_unixtime_to_localtime(time, tz=timezone)
key = 'used_percent' if check == 'memory' else key
value = r.get(check, {}).get(key, 0)
value = self.format_float(value)
server_list = data_lists.get(r_server_id)
server_list.append({"x": time, "y": value})
# Dont display empty lines
y_axis_sum = sum([x.get('y', 0) for x in data_lists[r_server_id]])
if y_axis_sum == 0:
data_lists[r_server_id] = []
series = []
for key, name in keys.items():
_index = list(keys.keys()).index(key)
data = [] if len(data_lists[key]) == 0 else data_lists[key]
chart_colors = select_colors(_index)
metric_type = {'name': name, 'data': data}
metric_type.update(chart_colors)
if len(data) > 0:
series.append(metric_type)
return series
def get_gauge_data_after(self,
enddate=None,
timestamp=None,
gauge=None,
timezone='UTC'):
if enddate:
result = self._get_gauge_data_for_period_query(timestamp=timestamp,
enddate=enddate,
gauge=gauge)
else:
result = self._get_gauge_data_after_query(timestamp, gauge=gauge)
gauge_keys = self.get_gauge_keys(gauge=gauge)
series = []
gauge_dict = {}
# Search and iterate over the data only if there are gauge keys
if gauge_keys:
for key in gauge_keys:
gauge_dict[key] = []
for r in result:
time = utc_unixtime_to_localtime(r['t'], tz=timezone)
for key in gauge_keys:
value = r.get(key, 0)
if not value:
value = r.get('value', 0)
gauge_dict[key].append({"x": time, "y": value})
# Eliminate empty lines
for key in gauge_keys:
y_axis_sum = sum([x.get('y', 0) for x in gauge_dict[key]])
if y_axis_sum == 0:
gauge_dict[key] = []
for i, key in enumerate(gauge_keys):
data = [] if len(gauge_dict[key]) == 0 else gauge_dict[key]
result = {
'color': colors[i],
'name': replace_underscore_with_dot_helper(key),
'data': data
}
if len(data) > 0:
series.append(result)
return series
def get_global_data_after(self, timestamp=None, metric=None, filtered_servers=None, timezone='UTC'):
gauge = metric.get('gauge')
key = metric.get('key')
plugin = metric.get('plugin')
keys = {}
data_lists = {}
for server in filtered_servers:
server_id = str(server['_id'])
keys[server_id] = server.get('name')
data_lists[server_id] = []
plugins = self.collection.find({'name': plugin})
for p in plugins:
p_server_id = str(p.get('server_id'))
server_list = data_lists.get(p_server_id)
result = None
gauge_cursor = self.get_gauge_by_name(plugin=p, name=gauge)
if gauge_cursor != None:
if key in gauge_cursor.get('keys'):
result = self._get_gauge_data_after_query(timestamp, gauge=gauge_cursor)
if result:
for r in result:
time = utc_unixtime_to_localtime(r['t'], tz=timezone)
value = r.get(key, None)
server_list.append({"x": time, "y": value})
series = []
for server, server_name in keys.items():
_index = keys.keys().index(server)
data = [] if len(data_lists[server]) == 0 else data_lists[server]
chart_colors = select_colors(_index)
result = {
'name': server_name,
'data': data
}
result.update(chart_colors)
if len(data) > 0:
series.append(result)
return series
def get_global_data_after(self, timestamp=None, metric=None, filtered_servers=None, timezone='UTC'):
gauge = metric.get('gauge')
key = metric.get('key')
plugin = metric.get('plugin')
keys = {}
data_lists = {}
for server in filtered_servers:
server_id = str(server['_id'])
keys[server_id] = server.get('name')
data_lists[server_id] = []
plugins = self.collection.find({'name': plugin})
for p in plugins:
p_server_id = str(p.get('server_id'))
server_list = data_lists.get(p_server_id)
result = None
gauge_cursor = self.get_gauge_by_name(plugin=p, name=gauge)
if gauge_cursor != None:
if key in gauge_cursor.get('keys'):
result = self._get_gauge_data_after_query(timestamp, gauge=gauge_cursor)
if result:
for r in result:
time = utc_unixtime_to_localtime(r['t'], tz=timezone)
value = r.get(key, None)
server_list.append({"x": time, "y": value})
series = []
for server, server_name in keys.items():
_index = keys.keys().index(server)
data = [] if len(data_lists[server]) == 0 else data_lists[server]
chart_colors = select_colors(_index)
result = {
'name': server_name,
'data': data
}
result.update(chart_colors)
if len(data) > 0:
series.append(result)
return series
def get_gauge_data_after(self, enddate=None, timestamp=None, gauge=None, timezone='UTC'):
if enddate:
result = self._get_gauge_data_for_period_query(timestamp=timestamp, enddate=enddate, gauge=gauge)
else:
result = self._get_gauge_data_after_query(timestamp, gauge=gauge)
gauge_keys = self.get_gauge_keys(gauge=gauge)
series = []
gauge_dict = {}
# Search and iterate over the data only if there are gauge keys
if gauge_keys:
for key in gauge_keys:
gauge_dict[key] = []
for r in result:
time = utc_unixtime_to_localtime(r['t'], tz=timezone)
for key in gauge_keys:
value = r.get(key, 0)
if not value:
value = r.get('value', 0)
gauge_dict[key].append({"x": time, "y": value})
# Eliminate empty lines
for key in gauge_keys:
y_axis_sum = sum([x.get('y', 0) for x in gauge_dict[key]])
if y_axis_sum == 0:
gauge_dict[key] = []
for i, key in enumerate(gauge_keys):
data = [] if len(gauge_dict[key]) == 0 else gauge_dict[key]
result = {
'color': colors[i],
'name': replace_underscore_with_dot_helper(key),
'data': data
}
if len(data) > 0:
series.append(result)
return series
def generate_charts(self,
keys=None,
result=None,
check=None,
timezone='UTC',
process=None):
keys_length = len(keys)
data_lists = []
data_lists = [[] for i in range(keys_length)]
for r in result:
data = r.get('data', [])
for d in data:
if d['p'] == process['_id']:
time = utc_unixtime_to_localtime(r['t'], tz=timezone)
for i in range(keys_length):
key = keys[i].key
value = d.get(key)
value = self.format_float(value)
data_lists[i].append({"x": time, "y": value})
for i in range(keys_length):
# Dont display empty lines
data_lists[i] = self.clear_y_axis(data=data_lists[i])
series = []
for i in range(keys_length):
data = [] if len(data_lists[i]) == 0 else data_lists[i]
metric_type = {
'color': colors[i],
'name': keys[i].name,
'data': data
}
series.append(metric_type)
return series
def generate_charts(self, keys=None, result=None, check=None, timezone='UTC', process=None):
keys_length = len(keys)
data_lists = []
data_lists = [[] for i in range(keys_length)]
for r in result:
data = r.get('data', [])
for d in data:
if d['p'] == process['_id']:
time = utc_unixtime_to_localtime(r['t'], tz=timezone)
for i in range(keys_length):
key = keys[i].key
value = d.get(key)
value = self.format_float(value)
data_lists[i].append({ "x": time, "y": value})
for i in range(keys_length):
# Dont display empty lines
data_lists[i] = self.clear_y_axis(data=data_lists[i])
series = []
for i in range(keys_length):
data = [] if len(data_lists[i]) == 0 else data_lists[i]
metric_type = {
'color': colors[i],
'name': keys[i].name,
'data': data
}
series.append(metric_type)
return series
def get(self):
date_from = self.get_argument('date_from', False)
date_to = self.get_argument('date_to', False)
charts = self.get_arguments('charts', None)
if date_from:
date_from = datestring_to_utc_datetime(date_from)
# Default - 24 hours period
else:
day = timedelta(hours=24)
date_from = self.now - day
if date_to:
date_to = datestring_to_utc_datetime(date_to)
else:
date_to = self.now
date_from = datetime_to_unixtime(date_from)
date_to = datetime_to_unixtime(date_to)
if len(charts) > 0:
active_checks = charts
else:
active_checks = settings.SYSTEM_CHECKS
checks = system_model.get_system_data(active_checks, date_from, date_to)
first_check_date = system_model.get_first_check_date()
# Convert the dates to local time for display
first_check_date = utc_unixtime_to_localtime(first_check_date)
date_from = utc_unixtime_to_localtime(date_from)
date_to = utc_unixtime_to_localtime(date_to)
# Get the difference between UTC and localtime - used to display
# the ticks in the charts
zone_difference = localtime_utc_timedelta()
# Get the max date - utc, converted to localtime
max_date = utc_now_to_localtime()
if checks != False:
network = []
network_interfaces = []
disk = []
volumes = []
# Add network adapters
if 'network' in active_checks:
for check in checks['network']:
network.append(check)
_interfaces = get_network_interfaces()
for interface in _interfaces:
if interface not in network_interfaces:
network_interfaces.append(interface)
# Add disk volumes
if 'disk' in active_checks:
for check in checks['disk']:
disk.append(check)
_volumes = get_disk_volumes()
for volume in _volumes:
if volume not in volumes:
volumes.append(volume)
self.render('system.html',
current_page='system',
active_checks=active_checks,
charts=charts,
checks=checks,
network=network,
network_interfaces=network_interfaces,
volumes=volumes,
disk=disk,
date_from=date_from,
date_to=date_to,
first_check_date=first_check_date,
zone_difference=zone_difference,
max_date=max_date
)
def dateformat_local(value, format='%d-%m-%Y-%H:%M'):
value = utc_unixtime_to_localtime(value)
return dateformat(value)
def dateformat_local(value, format="%d-%m-%Y-%H:%M"):
value = utc_unixtime_to_localtime(value)
return dateformat(value)
def get_global_data_after(self,
timestamp=None,
enddate=None,
key=None,
timezone='UTC',
check=None,
filtered_servers=None):
keys = {}
data_lists = {}
process_ids_list = []
series = []
server_ids_list = [x.get('_id') for x in filtered_servers]
filter_proceses_params = {
'name': key,
'server': {
"$in": server_ids_list
}
}
all_processes = self.collection.find(filter_proceses_params)
for process in all_processes:
process_id = str(process['_id'])
name = ''
server = filter(lambda x: x['_id'] == process['server'],
filtered_servers)
if len(server) > 0:
name = server[0].get('name')
keys[process_id] = name
data_lists[process_id] = []
process_ids_list.append(process_id)
params = {"t": {"$gte": int(timestamp)}}
if enddate:
params['t']["$lte"] = int(enddate)
params['server_id'] = {"$in": server_ids_list}
result = self.data_collection.find(params).sort('t', self.asc)
for r in result:
time = r.get('time') if r.get('time') else r.get('t')
time = utc_unixtime_to_localtime(time, tz=timezone)
data = r.get('data', [])
for d in data:
process_id = str(d['p'])
if process_id in process_ids_list:
time = utc_unixtime_to_localtime(r['t'], tz=timezone)
key = 'c' if check == 'cpu' else 'm'
value = d.get(key)
value = self.format_float(value)
data_lists[process_id].append({"x": time, "y": value})
for key, name in keys.items():
_index = keys.keys().index(key)
data = [] if len(data_lists[key]) == 0 else data_lists[key]
chart_colors = select_colors(_index)
metric_type = {'name': name, 'data': data}
metric_type.update(chart_colors)
if len(data) > 0:
series.append(metric_type)
return series
def get_global_device_data_after(self,
timestamp=None,
enddate=None,
check=None,
key=None,
timezone="UTC",
filtered_servers=None):
params = {"t": {"$gte": int(timestamp)}}
if enddate:
params['t']["$lte"] = int(enddate)
keys = {}
series = []
data_lists = {}
datamodel = volumes_model if check == 'disk' else interfaces_model
devices = datamodel.get_all_for_servers_list(servers=filtered_servers)
for device in devices:
data_collection = datamodel.get_data_collection()
params['device_id'] = device['_id']
result = data_collection.find(params).sort('t', self.asc)
if result.clone().count() > 0:
device_server_id = device['server_id']
device_server = None
for server in filtered_servers:
if server['_id'] == device_server_id:
device_server = server
# Server exists
if device_server != None:
_id = str(device['_id'])
keys[_id] = u"{server}.{device}".format(
server=device_server.get('name', ""),
device=device.get('name'))
data_lists[_id] = []
for r in result:
time = utc_unixtime_to_localtime(r.get('t', 0),
tz=timezone)
value = r.get(key, 0)
value = self.format_float(value)
data_lists[_id].append({"x": time, "y": value})
# Dont display empty lines
y_axis_sum = sum([x.get('y', 0) for x in data_lists[_id]])
if y_axis_sum == 0:
data_lists[_id] = []
for _id, name in keys.items():
_index = list(keys.keys()).index(_id)
data = [] if len(data_lists[_id]) == 0 else data_lists[_id]
chart_colors = select_colors(_index)
metric_type = {
'name': name,
'data': data,
}
metric_type.update(chart_colors)
if len(data) > 0:
series.append(metric_type)
return series
def get_global_data_after(self, timestamp=None, enddate=None, key=None, timezone='UTC', check=None, filtered_servers=None):
keys = {}
data_lists = {}
process_ids_list = []
series = []
server_ids_list = [x.get('_id') for x in filtered_servers]
filter_proceses_params = {'name': key, 'server': {"$in": server_ids_list}}
all_processes = self.collection.find(filter_proceses_params)
for process in all_processes:
process_id = str(process['_id'])
name = ''
server = filter(lambda x: x['_id'] == process['server'], filtered_servers)
if len(server) > 0:
name = server[0].get('name')
keys[process_id] = name
data_lists[process_id] = []
process_ids_list.append(process_id)
params = {"t": {"$gte": int(timestamp)}}
if enddate:
params['t']["$lte"] = int(enddate)
params['server_id'] = {"$in": server_ids_list}
result = self.data_collection.find(params).sort('t', self.asc)
for r in result:
time = r.get('time') if r.get('time') else r.get('t')
time = utc_unixtime_to_localtime(time, tz=timezone)
data = r.get('data', [])
for d in data:
process_id = str(d['p'])
if process_id in process_ids_list:
time = utc_unixtime_to_localtime(r['t'], tz=timezone)
key = 'c' if check == 'cpu' else 'm'
value = d.get(key)
value = self.format_float(value)
data_lists[process_id].append({ "x": time, "y": value})
for key, name in keys.items():
_index = keys.keys().index(key)
data = [] if len(data_lists[key]) == 0 else data_lists[key]
chart_colors = select_colors(_index)
metric_type = {
'name': name,
'data': data
}
metric_type.update(chart_colors)
if len(data) > 0:
series.append(metric_type)
return series
def get(self):
date_from = self.get_argument('date_from', False)
date_to = self.get_argument('date_to', False)
charts = self.get_arguments('charts', None)
if date_from:
date_from = datestring_to_utc_datetime(date_from)
# Default - 24 hours period
else:
day = timedelta(hours=24)
date_from = self.now - day
if date_to:
date_to = datestring_to_utc_datetime(date_to)
else:
date_to = self.now
date_from = datetime_to_unixtime(date_from)
date_to = datetime_to_unixtime(date_to)
if len(charts) > 0:
active_checks = charts
else:
active_checks = settings.SYSTEM_CHECKS
checks = system_model.get_system_data(active_checks, date_from,
date_to)
first_check_date = system_model.get_first_check_date()
# Convert the dates to local time for display
first_check_date = utc_unixtime_to_localtime(first_check_date)
date_from = utc_unixtime_to_localtime(date_from)
date_to = utc_unixtime_to_localtime(date_to)
# Get the difference between UTC and localtime - used to display
# the ticks in the charts
zone_difference = localtime_utc_timedelta()
# Get the max date - utc, converted to localtime
max_date = utc_now_to_localtime()
if checks != False:
network = []
network_interfaces = []
disk = []
volumes = []
# Add network adapters
if 'network' in active_checks:
for check in checks['network']:
network.append(check)
_interfaces = get_network_interfaces()
for interface in _interfaces:
if interface not in network_interfaces:
network_interfaces.append(interface)
# Add disk volumes
if 'disk' in active_checks:
for check in checks['disk']:
disk.append(check)
_volumes = get_disk_volumes()
for volume in _volumes:
if volume not in volumes:
volumes.append(volume)
self.render('system.html',
current_page='system',
active_checks=active_checks,
charts=charts,
checks=checks,
network=network,
network_interfaces=network_interfaces,
volumes=volumes,
disk=disk,
date_from=date_from,
date_to=date_to,
first_check_date=first_check_date,
zone_difference=zone_difference,
max_date=max_date)
