def prune_old_keys():
for key in get_all_data('pingdom_*'):
if key.split('pingdom_')[1] not in pingdom_keys:
delete_data(key)
for key in get_all_data('newrelic_*'):
if key.split('newrelic_')[1] not in newrelic_keys:
delete_data(key)
def list_unreporting_servers():
found_servers = set()
for key in get_all_data('resources:*'):
host_data = json.loads(get_data(key))[0]
found_servers.add(host_data['name'])
reporting_servers = set()
tick_data, tick_data_validity = get_tick_data()
for host in tick_data:
host_data = tick_data[host]
reporting_servers.add(host_data['name'])
newrelic_servers_data, newrelic_servers_data_validity = get_newrelic_servers_data(
)
for host in newrelic_servers_data:
host_data = newrelic_servers_data[host]
reporting_servers.add(host_data['name'])
newrelic_infra_data, newrelic_infra_data_validity = get_newrelic_infra_data(
)
for host in newrelic_infra_data:
host_data = newrelic_infra_data[host]
reporting_servers.add(host_data['name'])
return found_servers - reporting_servers
def get_calendar_items():
calendar_items = []
calendar_keys = sorted(get_all_data('calendar_*')) # Get all the calendar keys from Redis
for key in calendar_keys:
old_date = key.replace('calendar_', '')
convert = datetime.datetime.strptime(old_date, '%Y-%m-%d') # Convert the date to a nice format for the Warboard
calendar_items.append({convert.strftime('%a %d %B'): get_data(key)})
return(calendar_items)
def get_calendar_items():
calendar_items = []
calendar_keys = sorted(
get_all_data('calendar_*')) # Get all the calendar keys from Redis
for key in calendar_keys:
old_date = key.replace('calendar_', '')
convert = datetime.datetime.strptime(
old_date,
'%Y-%m-%d') # Convert the date to a nice format for the Warboard
calendar_items.append({convert.strftime('%a %d %B'): get_data(key)})
return (calendar_items)
def get_resource_results():
"""
Merges lists returned by resource modules into one list in the correct
format for warboard.html to display monitored resources
{% for check in resource_results['checks']|sort(attribute='orderby')|reverse %}
<tr class="danger lead"><td>{{ check['name'] }}</td><td>{{ check['summary']['cpu'] }}%</td><td>{{ check['summary']['memory'] }}%</td><td>{{ check['summary']['disk_io'] }}%</td><td>{{ check['summary']['fullest_disk'] }}%</td></tr>
"""
resource_results = {}
resource_results['checks'] = []
resource_results['green'] = 0
resource_results['red'] = 0
resource_results['orange'] = 0
resource_results['blue'] = 0
resource_results['failed_accounts'] = 0
resource_results['total_accounts'] = 0
resource_results['total_checks'] = 0
successful_checks = 0
# Defaults for when no data is reported, working towards having modules be
# modular / optional
resource_results['blue_percent'] = 100
resource_results['red_percent'] = 0
resource_results['orange_percent'] = 0
resource_results['green_percent'] = 0
resource_results['working_percentage'] = 100
# Check if the data recieved from each module is still valid, if it is not
# then all checks from that module are counted as unsuccessful and all
# accounts are counted as failed
milliseconds_since_epoch = time.time() * 1000
for module in get_all_data('resources_success:*'):
module_success_json = get_data(module)
module_success = json.loads(module_success_json)[0]
resource_results['total_accounts'] += module_success['total_accounts']
resource_results['total_checks'] += module_success['total_checks']
milliseconds_since_epoch_module_data_is_valid_until = module_success[
'valid_until']
if milliseconds_since_epoch > milliseconds_since_epoch_module_data_is_valid_until:
resource_results['failed_accounts'] += module_success[
'total_accounts']
else:
resource_results['failed_accounts'] += module_success[
'failed_accounts']
successful_checks += module_success['successful_checks']
resource_results[
'failed_checks'] = resource_results['total_checks'] - successful_checks
checks_found = 0
# Get list of keys in the format resources:module#uuid
for host in get_all_data('resources:*'):
try:
# Storing lists with only one value since when I convert dictionarys
# to json and store them in redis they come back as strings, I am
# working around this by storing lists, ast.literal_eval also works
host_data = json.loads(get_data(host))[0]
resource_results['checks'].append(host_data)
checks_found += 1
# get the health status colour of the current check, and then add
# one to the number of checks with that health status
resource_results[host_data['health_status']] += 1
except Exception as e:
resource_results['failed_checks'] += 1
# I would rather log to uwsgi's log but I'll sort this out later
log_messages(
'Data for {} is not in a valid format: {}'.format(host, e),
'error')
# If we are getting back old checks that are no-longer reporting hence
# are not in the total_checks variable then they have failed.
# If we are getting back less checks than we stored then something has
# gone really wrong or we caught the weekly cron that clears the keys.
resource_results['failed_checks'] += abs(resource_results['total_checks'] -
checks_found)
resource_results['total_checks'] = checks_found
total_results = resource_results['green'] + resource_results[
'red'] + resource_results['orange'] + resource_results['blue']
if total_results != 0:
resource_results['red_percent'] = (resource_results['red'] /
total_results) * 100
resource_results['orange_percent'] = (resource_results['orange'] /
total_results) * 100
resource_results['blue_percent'] = (resource_results['blue'] /
total_results) * 100
# I want the percentage to always be 100 and green seems the most
# disposable / least affected by any rounding issues
resource_results['green_percent'] = 100 - (
resource_results['red_percent'] +
resource_results['orange_percent'] +
resource_results['blue_percent'])
# Set the working percentage to the lowest of accounts and checks, if either
# have a total of 0 then resources isn't working so the working percentage
# can be set to 0 to avoid dividing by 0
if resource_results['total_accounts'] != 0 and resource_results[
'total_checks'] != 0:
accounts_working_percentage = 100 - (
(resource_results['failed_accounts'] /
resource_results['total_accounts']) * 100)
if accounts_working_percentage < resource_results['working_percentage']:
resource_results[
'working_percentage'] = accounts_working_percentage
checks_working_percentage = 100 - (
(resource_results['failed_checks'] /
resource_results['total_checks']) * 100)
if checks_working_percentage < resource_results['working_percentage']:
resource_results['working_percentage'] = checks_working_percentage
else:
resource_results['working_percentage'] = 0
resource_results['working_accounts'] = resource_results[
'total_accounts'] - resource_results['failed_accounts']
return resource_results
def clear_resources_keys():
for key in get_all_data('resources:*'):
delete_data(key)
def prune_calendar_items():
# We are now pruning all calendar items when reloading the calendar file to remove old entries we don't want.
calendar_keys = get_all_data('calendar_*')
for key in calendar_keys:
delete_data(key)
def prune_old_keys():
for key in get_all_data('pingdom_*'):
if key.split('pingdom_')[1] not in pingdom_keys:
delete_data(key)
def prune_calendar_items():
# We are now pruning all calendar items when reloading the calendar file to remove old entries we don't want.
calendar_keys = get_all_data('calendar_*')
for key in calendar_keys:
delete_data(key)