def get_pingdom_results():
all_results = []
pingdom_results = {}
for account in pingdom_keys:
result_json = json.loads(get_data('pingdom_'+account)) # Load all the pingdom keys as json and store them in the all_results list
all_results.append(result_json['checks'])
pingdom_results['pingdom_up'] = 0
pingdom_results['pingdom_down'] = 0
pingdom_results['pingdom_paused'] = 0
pingdom_results['total_pingdom_accounts'] = int(get_data('total_pingdom_accounts'))
pingdom_results['failed_pingdom'] = int(get_data('failed_pingdom'))
pingdom_results['working_pingdom'] = pingdom_results['total_pingdom_accounts']-pingdom_results['failed_pingdom']
pingdom_results['working_percentage'] = int(float(pingdom_results['working_pingdom'])/float(pingdom_results['total_pingdom_accounts'])*100)
pingdom_results['checks'] = chain_results(all_results) # Chain all the results together to be returned for the warboard
pingdom_results['total_checks'] = len(pingdom_results['checks'])
for check in pingdom_results['checks']: # Categorize all the checks as up/down etc
if check['type'] == 'httpcustom':
check['type'] = 'custom'
if check['status'] == 'up':
pingdom_results['pingdom_up'] +=1
elif check['status'] == 'down':
pingdom_results['pingdom_down'] +=1
elif check['status'] == 'paused':
pingdom_results['pingdom_paused'] +=1
elif check['status'] == 'unknown':
check['status'] = 'paused'
check['lastresponsetime'] = 0
pingdom_results['down_percent'] = math.ceil(100*float(pingdom_results['pingdom_down'])/float(pingdom_results['total_checks']))
pingdom_results['paused_percent'] = math.ceil(100*float(pingdom_results['pingdom_paused'])/float(pingdom_results['total_checks']))
pingdom_results['up_percent'] = 100-pingdom_results['down_percent']-pingdom_results['paused_percent']
return(pingdom_results)
def get_pingdom_results():
all_results = []
pingdom_results = {}
for account in pingdom_keys:
result_json = json.loads(get_data('pingdom_'+account)) # Load all the pingdom keys as json and store them in the all_results list
all_results.append(result_json['checks'])
pingdom_results['pingdom_up'] = 0
pingdom_results['pingdom_down'] = 0
pingdom_results['pingdom_paused'] = 0
pingdom_results['total_pingdom_accounts'] = int(get_data('total_pingdom_accounts'))
pingdom_results['failed_pingdom'] = int(get_data('failed_pingdom'))
pingdom_results['working_pingdom'] = pingdom_results['total_pingdom_accounts']-pingdom_results['failed_pingdom']
pingdom_results['working_percentage'] = int(float(pingdom_results['working_pingdom'])/float(pingdom_results['total_pingdom_accounts'])*100)
pingdom_results['checks'] = chain_results(all_results) # Chain all the results together to be returned for the warboard
pingdom_results['total_checks'] = len(pingdom_results['checks'])
for check in pingdom_results['checks']: # Categorize all the checks as up/down etc
check['name'] = check['name'][:40] # Limit pingdom server names to 40 characters to not break the warboard layout
if check['type'] == 'httpcustom':
check['type'] = 'custom'
if check['status'] == 'up':
pingdom_results['pingdom_up'] +=1
elif check['status'] == 'down':
pingdom_results['pingdom_down'] +=1
elif check['status'] == 'paused':
pingdom_results['pingdom_paused'] +=1
if not 'lastresponsetime' in check:
check['lastresponsetime'] = 0
elif check['status'] == 'unknown':
check['status'] = 'paused'
check['lastresponsetime'] = 0
pingdom_results['down_percent'] = math.ceil(100*float(pingdom_results['pingdom_down'])/float(pingdom_results['total_checks']))
pingdom_results['paused_percent'] = math.ceil(100*float(pingdom_results['pingdom_paused'])/float(pingdom_results['total_checks']))
pingdom_results['up_percent'] = 100-pingdom_results['down_percent']-pingdom_results['paused_percent']
return(pingdom_results)
def get_sirportly_results(): # Get all the sirportly data to pass to the warboard, some things need to be ints for Jinja2 to do calcs
sirportly_results = {'users': {}}
sirportly_results['unassigned_tickets'] = int(get_data('unassigned_tickets'))
sirportly_results['resolved_tickets'] = int(get_data('resolved_tickets'))
sirportly_results['red_percent'] = get_data('red_percent')
sirportly_results['green_percent'] = get_data('green_percent')
sirportly_results['multiplier'] = int(get_data('multiplier'))
sirportly_results['red_tickets'] = get_data('red_tickets')
sirportly_results['total_tickets'] = get_data('total_tickets')
for user in sirportly_users:
sirportly_results['users'][user+'_green'] = int(get_data(user+'_green'))
sirportly_results['users'][user+'_red'] = int(get_data(user+'_red'))
sirportly_results['users'][user+'_total'] = get_data(user+'_total')
return(sirportly_results)
def store_calendar_items():
with open(calendar_export) as c_file:
try:
c_data = json.load(c_file)
except ValueError:
c_data = False
c_file.close()
if c_data != False:
prune_calendar_items()
for item in c_data['items']:
if 'dateTime' in item['start']: # Check if the datetime is set
item['start']['date'] = item['start']['dateTime'].split('T')[0] # Split the datetime to get the date and set the data parameter
current_summary = item['summary']
try:
start_time = datetime.datetime.strptime(item['start']['dateTime'].split('T')[1], '%H:%M:%SZ').strftime('%H:%M') # Convert the start time to a nice date
end_time = datetime.datetime.strptime(item['end']['dateTime'].split('T')[1], '%H:%M:%SZ').strftime('%H:%M: ') # Convert the end time to a nice date
except ValueError:
start_time = datetime.datetime.strptime(item['start']['dateTime'].split('T')[1], '%H:%M:%S+01:00').strftime('%H:%M') # To work with DST times
end_time = datetime.datetime.strptime(item['end']['dateTime'].split('T')[1], '%H:%M:%S+01:00').strftime('%H:%M: ')
item['summary'] = start_time+' - '+end_time+current_summary # Add the start and end time to the summary
current = get_data('calendar_'+item['start']['date']) # Check if an existing key exists for the date in question
if current == None:
set_data('calendar_'+item['start']['date'], item['summary']) # If a date doesn't exist create one
elif item['summary'] not in current: # If a key exists but it's not the current summary it means we have two items for one date
set_data('calendar_'+item['start']['date'], current+calendar_split+item['summary']) # Append to the existing item
else:
log_messages('Could not parse calendar', 'error')
def list_unreporting_servers():
found_servers = set()
for key in get_all_data('resources:*'):
host_data = json.loads(get_data(key))[0]
found_servers.add(host_data['name'])
reporting_servers = set()
tick_data, tick_data_validity = get_tick_data()
for host in tick_data:
host_data = tick_data[host]
reporting_servers.add(host_data['name'])
newrelic_servers_data, newrelic_servers_data_validity = get_newrelic_servers_data(
)
for host in newrelic_servers_data:
host_data = newrelic_servers_data[host]
reporting_servers.add(host_data['name'])
newrelic_infra_data, newrelic_infra_data_validity = get_newrelic_infra_data(
)
for host in newrelic_infra_data:
host_data = newrelic_infra_data[host]
reporting_servers.add(host_data['name'])
return found_servers - reporting_servers
def get_calendar_items():
calendar_items = []
calendar_keys = sorted(get_all_data('calendar_*')) # Get all the calendar keys from Redis
for key in calendar_keys:
old_date = key.replace('calendar_', '')
convert = datetime.datetime.strptime(old_date, '%Y-%m-%d') # Convert the date to a nice format for the Warboard
calendar_items.append({convert.strftime('%a %d %B'): get_data(key)})
return(calendar_items)
def get_calendar_items():
calendar_items = []
calendar_keys = sorted(
get_all_data('calendar_*')) # Get all the calendar keys from Redis
for key in calendar_keys:
old_date = key.replace('calendar_', '')
convert = datetime.datetime.strptime(
old_date,
'%Y-%m-%d') # Convert the date to a nice format for the Warboard
calendar_items.append({convert.strftime('%a %d %B'): get_data(key)})
return (calendar_items)
def get_newrelic_results():
all_results = []
newrelic_results = {}
for account in newrelic_keys:
result_json = json.loads(get_data('newrelic_'+account)) # Pull the NR data from redis load it as json and append to a list
all_results.append(result_json['servers'])
newrelic_results['total_newrelic_accounts'] = int(get_data('total_newrelic_accounts'))
newrelic_results['failed_newrelic'] = int(get_data('failed_newrelic'))
newrelic_results['working_newrelic'] = newrelic_results['total_newrelic_accounts']-newrelic_results['failed_newrelic']
newrelic_results['working_percentage'] = int(float(newrelic_results['working_newrelic'])/float(newrelic_results['total_newrelic_accounts'])*100)
newrelic_results['checks'] = chain_results(all_results) # Store all the nr results as 1 chained list
newrelic_results['total_checks'] = len(newrelic_results['checks'])
newrelic_results['green'] = 0
newrelic_results['red'] = 0
newrelic_results['orange'] = 0
newrelic_results['blue'] = 0
for check in newrelic_results['checks']: # Categorize all the checks as up/down and use the highest metric for each item as the thing we order by
check['name'] = check['name'][:30] # Limit newrelic server names to 30 characters to not break the warboard layout
if check['reporting'] == True:
check['orderby'] = max(check['summary']['cpu'], check['summary']['memory'], check['summary']['fullest_disk'], check['summary']['disk_io'])
if check['health_status'] == 'green':
newrelic_results['green'] +=1
elif check['health_status'] == 'orange':
newrelic_results['orange'] +=1
elif check['health_status'] == 'red':
newrelic_results['red'] +=1
elif check['reporting'] == False:
check['orderby'] = 0
check['health_status'] = 'blue'
newrelic_results['blue'] +=1
newrelic_results['red_percent'] = math.ceil(100*float(newrelic_results['red'])/float(newrelic_results['total_checks']))
newrelic_results['green_percent'] = math.ceil(100*float(newrelic_results['green'])/float(newrelic_results['total_checks']))
newrelic_results['orange_percent'] = math.ceil(100*float(newrelic_results['orange'])/float(newrelic_results['total_checks']))
newrelic_results['blue_percent'] = 100-newrelic_results['red_percent']-newrelic_results['green_percent']-newrelic_results['orange_percent']
if newrelic_results['blue_percent'] < 0:
newrelic_results['green_percent'] = newrelic_results['green_percent']-abs(newrelic_results['blue_percent'])
return(newrelic_results)
def store_calendar_items():
with open(calendar_export) as c_file:
try:
c_data = json.load(c_file)
except ValueError:
c_data = False
c_file.close()
if c_data != False:
prune_calendar_items()
for item in c_data['items']:
if 'dateTime' in item['start']: # Check if the datetime is set
item['start']['date'] = item['start']['dateTime'].split(
'T'
)[0] # Split the datetime to get the date and set the data parameter
current_summary = item['summary']
try:
start_time = datetime.datetime.strptime(
item['start']['dateTime'].split('T')[1],
'%H:%M:%SZ').strftime(
'%H:%M') # Convert the start time to a nice date
end_time = datetime.datetime.strptime(
item['end']['dateTime'].split('T')[1],
'%H:%M:%SZ').strftime(
'%H:%M: ') # Convert the end time to a nice date
except ValueError:
start_time = datetime.datetime.strptime(
item['start']['dateTime'].split('T')[1],
'%H:%M:%S+01:00').strftime(
'%H:%M') # To work with DST times
end_time = datetime.datetime.strptime(
item['end']['dateTime'].split('T')[1],
'%H:%M:%S+01:00').strftime('%H:%M: ')
item[
'summary'] = start_time + ' - ' + end_time + current_summary # Add the start and end time to the summary
current = get_data(
'calendar_' + item['start']['date']
) # Check if an existing key exists for the date in question
if current == None:
set_data('calendar_' + item['start']['date'],
item['summary']) # If a date doesn't exist create one
elif item[
'summary'] not in current: # If a key exists but it's not the current summary it means we have two items for one date
set_data('calendar_' + item['start']['date'],
current + calendar_split +
item['summary']) # Append to the existing item
else:
log_messages('Could not parse calendar', 'error')
def get_resource_results():
"""
Merges lists returned by resource modules into one list in the correct
format for warboard.html to display monitored resources
{% for check in resource_results['checks']|sort(attribute='orderby')|reverse %}
<tr class="danger lead"><td>{{ check['name'] }}</td><td>{{ check['summary']['cpu'] }}%</td><td>{{ check['summary']['memory'] }}%</td><td>{{ check['summary']['disk_io'] }}%</td><td>{{ check['summary']['fullest_disk'] }}%</td></tr>
"""
resource_results = {}
resource_results['checks'] = []
resource_results['green'] = 0
resource_results['red'] = 0
resource_results['orange'] = 0
resource_results['blue'] = 0
resource_results['failed_accounts'] = 0
resource_results['total_accounts'] = 0
resource_results['total_checks'] = 0
successful_checks = 0
# Defaults for when no data is reported, working towards having modules be
# modular / optional
resource_results['blue_percent'] = 100
resource_results['red_percent'] = 0
resource_results['orange_percent'] = 0
resource_results['green_percent'] = 0
resource_results['working_percentage'] = 100
# Check if the data recieved from each module is still valid, if it is not
# then all checks from that module are counted as unsuccessful and all
# accounts are counted as failed
milliseconds_since_epoch = time.time() * 1000
for module in get_all_data('resources_success:*'):
module_success_json = get_data(module)
module_success = json.loads(module_success_json)[0]
resource_results['total_accounts'] += module_success['total_accounts']
resource_results['total_checks'] += module_success['total_checks']
milliseconds_since_epoch_module_data_is_valid_until = module_success[
'valid_until']
if milliseconds_since_epoch > milliseconds_since_epoch_module_data_is_valid_until:
resource_results['failed_accounts'] += module_success[
'total_accounts']
else:
resource_results['failed_accounts'] += module_success[
'failed_accounts']
successful_checks += module_success['successful_checks']
resource_results[
'failed_checks'] = resource_results['total_checks'] - successful_checks
checks_found = 0
# Get list of keys in the format resources:module#uuid
for host in get_all_data('resources:*'):
try:
# Storing lists with only one value since when I convert dictionarys
# to json and store them in redis they come back as strings, I am
# working around this by storing lists, ast.literal_eval also works
host_data = json.loads(get_data(host))[0]
resource_results['checks'].append(host_data)
checks_found += 1
# get the health status colour of the current check, and then add
# one to the number of checks with that health status
resource_results[host_data['health_status']] += 1
except Exception as e:
resource_results['failed_checks'] += 1
# I would rather log to uwsgi's log but I'll sort this out later
log_messages(
'Data for {} is not in a valid format: {}'.format(host, e),
'error')
# If we are getting back old checks that are no-longer reporting hence
# are not in the total_checks variable then they have failed.
# If we are getting back less checks than we stored then something has
# gone really wrong or we caught the weekly cron that clears the keys.
resource_results['failed_checks'] += abs(resource_results['total_checks'] -
checks_found)
resource_results['total_checks'] = checks_found
total_results = resource_results['green'] + resource_results[
'red'] + resource_results['orange'] + resource_results['blue']
if total_results != 0:
resource_results['red_percent'] = (resource_results['red'] /
total_results) * 100
resource_results['orange_percent'] = (resource_results['orange'] /
total_results) * 100
resource_results['blue_percent'] = (resource_results['blue'] /
total_results) * 100
# I want the percentage to always be 100 and green seems the most
# disposable / least affected by any rounding issues
resource_results['green_percent'] = 100 - (
resource_results['red_percent'] +
resource_results['orange_percent'] +
resource_results['blue_percent'])
# Set the working percentage to the lowest of accounts and checks, if either
# have a total of 0 then resources isn't working so the working percentage
# can be set to 0 to avoid dividing by 0
if resource_results['total_accounts'] != 0 and resource_results[
'total_checks'] != 0:
accounts_working_percentage = 100 - (
(resource_results['failed_accounts'] /
resource_results['total_accounts']) * 100)
if accounts_working_percentage < resource_results['working_percentage']:
resource_results[
'working_percentage'] = accounts_working_percentage
checks_working_percentage = 100 - (
(resource_results['failed_checks'] /
resource_results['total_checks']) * 100)
if checks_working_percentage < resource_results['working_percentage']:
resource_results['working_percentage'] = checks_working_percentage
else:
resource_results['working_percentage'] = 0
resource_results['working_accounts'] = resource_results[
'total_accounts'] - resource_results['failed_accounts']
return resource_results
def get_sirportly_results(
): # Get all the sirportly data to pass to the warboard, some things need to be ints for Jinja2 to do calcs
sirportly_results = {'users': {}}
sirportly_results['unassigned_tickets'] = int(
get_data('unassigned_tickets'))
sirportly_results['resolved_tickets'] = int(get_data('resolved_tickets'))
sirportly_results['red_percent'] = get_data('red_percent')
sirportly_results['green_percent'] = get_data('green_percent')
sirportly_results['multiplier'] = float(get_data('multiplier'))
sirportly_results['red_tickets'] = get_data('red_tickets')
sirportly_results['total_tickets'] = get_data('total_tickets')
for user in sirportly_users:
sirportly_results['users'][user + '_red'] = int(get_data(user +
'_red'))
sirportly_results['users'][user + '_green'] = int(
get_data(user + '_green'))
sirportly_results['users'][user + '_blue'] = int(
get_data(user + '_blue'))
sirportly_results['users'][user + '_total'] = get_data(user + '_total')
return (sirportly_results)