def get_current_system_information_for_all_services():
return {
'server':
system_data_service.get_system_information(),
'denva':
local_data_gateway.get_data_for('{}/system'.format(
config_service.load_cfg()["urls"]['denva'])),
'enviro':
local_data_gateway.get_data_for('{}/system'.format(
config_service.load_cfg()["urls"]['enviro'])),
'delight':
local_data_gateway.get_data_for('{}/system'.format(
config_service.load_cfg()["urls"]['delight']))
}
def set_denva_status(cfg):
# 1. DENVA
state = status.Status()
logger.info('Getting status for denva..')
server_data = local_data_gateway.get_data_for('{}/system'.format(
config_service.load_cfg()["urls"]['denva']))
if 'error' in server_data:
logger.warning('Unable to get Denva status due to {}'.format(
server_data['error']))
state.set_error()
else:
system_health_check_service.update_hc_for('denva', 'ui')
if float(dom_utils.get_float_number_from_text(
server_data['CPU Temp'])) > cfg['sensor']['cpu_temp_error']:
logger.warning('status: RED due to very high cpu temp on Denva )')
state.set_error()
elif float(
dom_utils.get_float_number_from_text(
server_data['CPU Temp'])) > cfg['sensor']['cpu_temp_warn']:
logger.warning('status: ORANGE due to high cpu temp on Denva )')
state.set_warn()
if dom_utils.get_int_number_from_text(
server_data['Memory Available']) < 384:
logger.warning(
'status: RED due to very low memory available on Denva')
state.set_error()
elif dom_utils.get_int_number_from_text(
server_data['Memory Available']) < 512:
logger.warning(
'status: ORANGE due to low memory available on Denva')
state.set_warn()
if dom_utils.get_int_number_from_text(server_data['Free Space']) < 256:
logger.warning('status: RED due to very low free space on Denva')
state.set_error()
elif dom_utils.get_int_number_from_text(
server_data['Free Space']) < 1024:
logger.warning('status: ORANGE due to low free space on Denva')
state.set_warn()
if dom_utils.get_int_number_from_text(
server_data['Data Free Space']) < 256:
logger.warning(
'status: RED due to very low data free space on Denva')
state.set_error()
elif dom_utils.get_int_number_from_text(
server_data['Data Free Space']) < 1024:
logger.warning(
'status: ORANGE due to low data free space on Denva')
state.set_warn()
color_red, color_green, color_blue = delight_utils.get_state_colour(state)
update_blink(state.state)
unicornhathd.set_pixel(ui_utils.to_x(1), 1, purple_r, purple_g, purple_b)
set_status_for_device(1, 13, color_red, color_green, color_blue)
logger.info('Denva: {}'.format(state.get_status_as_light_colour()))
return blink
def get_hc_for_radar():
response = {'dump': 'DOWN', 'digest': commands.is_dump_digest_active()}
dump_response = local_data_gateway.get_data_for(
config_service.get_radar_hc_url(), 2)
if 'error' in dump_response:
logger.warning(dump_response['error'])
else:
response['dump'] = 'UP'
return response
def reboot_all_devices():
local_data_gateway.get_data_for('{}/reboot'.format(
config_service.load_cfg()["urls"]['denva']))
local_data_gateway.get_data_for('{}/reboot'.format(
config_service.load_cfg()["urls"]['enviro']))
local_data_gateway.get_data_for('{}/reboot'.format(
config_service.load_cfg()["urls"]['delight']))
return {'result': 'All devices starting to reboot'}
def stop_all_devices():
local_data_gateway.get_data_for('{}/halt'.format(
config_service.load_cfg()["urls"]['denva']))
local_data_gateway.get_data_for('{}/halt'.format(
config_service.load_cfg()["urls"]['enviro']))
local_data_gateway.get_data_for('{}/halt'.format(
config_service.load_cfg()["urls"]['delight']))
return {'result': 'All devices stopped'}
def set_delight_status(cfg):
# 4. DELIGHT
state = status.Status()
try:
delight_data = delight_service.get_system_info()
cpu_temp = str(delight_data['CPU Temp'])
if float(dom_utils.get_float_number_from_text(
cpu_temp)) > cfg['sensor']['cpu_temp_error']:
logger.warning('status: RED due to very high cpu temp on Delight')
state.set_error()
elif float(dom_utils.get_float_number_from_text(
cpu_temp)) > cfg['sensor']['cpu_temp_warn']:
logger.warning('status: ORANGE due to high cpu temp on Delight')
state.set_warn()
if dom_utils.get_int_number_from_text(
delight_data['Memory Available']) < 128:
logger.warning(
'status: RED due to very low memory available on Delight')
state.set_error()
elif dom_utils.get_int_number_from_text(
delight_data['Memory Available']) < 256:
logger.warning(
'status: ORANGE due to low memory available on Delight')
state.set_warn()
if dom_utils.get_int_number_from_text(
delight_data['Free Space']) < 128:
logger.warning('status: RED due to very low free space on Delight')
state.set_error()
elif dom_utils.get_int_number_from_text(
delight_data['Free Space']) < 512:
logger.warning('status: ORANGE due to low free space on Delight')
state.set_warn()
except Exception as set_state_exception:
logger.error(
'Something went badly wrong\n{}'.format(set_state_exception),
exc_info=True)
state.set_error()
delight_ui_response = local_data_gateway.get_data_for('{}/hc'.format(
config_service.load_cfg()["urls"]['delight']))
if not 'error' in delight_ui_response:
system_health_check_service.update_hc_for('delight', 'ui')
color_blue, color_green, color_red = delight_utils.get_state_colour(state)
update_blink(state.state)
unicornhathd.set_pixel(ui_utils.to_x(13), 1, purple_r, purple_g, purple_b)
unicornhathd.set_pixel(ui_utils.to_x(15), 1, purple_r, purple_g, purple_b)
unicornhathd.set_pixel(ui_utils.to_x(13), 2, purple_r, purple_g, purple_b)
unicornhathd.set_pixel(ui_utils.to_x(15), 2, purple_r, purple_g, purple_b)
set_status_for_device(13, 13, color_red, color_green, color_blue)
logger.info('Delight: {}'.format(state.get_status_as_light_colour()))
def test_get_data_for_non_working_url_should_return_error(self):
if config_service.run_slow_test():
# given
url = 'http://192.168.0.204:5000/system'
# when
result = local_data_gateway.get_data_for(url)
# then
if 'error' not in result:
self.fail()
else:
self.skipTest(
'running fast test only. local_data_gateway.get_data_for skipped.'
)
def digest():
# load and save them to file for stats
counter = 0
errors = 0 # add when error was happen last time
warnings = 0
while True:
counter += 1
start_time = timer()
result = local_data_gateway.get_data_for(
config_service.get_url_for_dump1090(), 5)
if 'error' in result:
logger.error(result['error'])
local_data_gateway.post_metrics_update('flight', 'errors')
errors += 1
logger.error('Errors: {}'.format(errors))
else:
aircraft_storage.save_raw_reading(result)
aircraft_storage.save_processed_data(result)
if counter % 2 == 0:
local_data_gateway.post_healthcheck_beat('other', 'radar')
local_data_gateway.post_metrics_update('flight', 'ok')
end_time = timer()
measurement = int((end_time - start_time) * 1000)
measurement_time = str(measurement) # in ms
if measurement > config_service.max_latency():
warnings += 1
logger.warning("Measurement {} was slow.It took {} ms".format(
counter, measurement))
if counter % 2 == 0:
local_data_gateway.post_healthcheck_beat('other', 'digest')
display_stats()
measurement_message = 'Measurement no. {} It took {} milliseconds to process. Errors: {}. Warnings: {}'.format(
counter, measurement_time, errors, warnings)
logger.info(measurement_message)
remaining_time = refresh_rate_in_seconds - (float(measurement_time) /
1000)
if remaining_time > 0:
time.sleep(remaining_time)
def create_for_current_measurements():
return {
'information':
information.get_data_about_rickmansworth(),
'denva':
local_data_gateway.get_current_reading_for_denva(),
'enviro':
local_data_gateway.get_current_reading_for_enviro(),
'aircraft':
local_data_gateway.get_current_reading_for_aircraft(),
'warnings':
local_data_gateway.get_current_warnings_for_all_services(),
'logs':
local_data_gateway.get_current_logs_for_all_services(),
'system':
app_server_service.get_current_system_information_for_all_services(),
'status':
local_data_gateway.get_data_for('http://192.168.0.203:5000/shc/get', 3)
}
def generate():
logger.info('Preparing to send denva report email')
start_time = timer()
email_data = {
'now': {
'denva': local_data_gateway.get_current_reading_for_denva(),
'enviro': local_data_gateway.get_current_reading_for_enviro(),
'aircraft': local_data_gateway.get_current_reading_for_aircraft()
},
'report': {
'denva': local_data_gateway.get_yesterday_report_for_denva(),
'enviro': local_data_gateway.get_yesterday_report_for_enviro(),
'aircraft': local_data_gateway.get_yesterday_report_for_aircraft(),
'rickmansworth':
information_service.get_data_about_rickmansworth(),
},
'status':
local_data_gateway.get_data_for('http://192.168.0.203:5000/shc/get', 3)
}
end_time = timer()
logger.info('It took {} ms to generate data'.format(
int((end_time - start_time) * 1000)))
return email_data
def set_mothership_status():
# 3. MOTHERSHIP SERVER
state = status.Status()
server_data = local_data_gateway.get_data_for('{}/system'.format(
config_service.load_cfg()["urls"]['server']))
if 'error' in server_data:
logger.warning('Unable to get Server status due to {}'.format(
server_data['error']))
state.set_error()
else:
system_health_check_service.update_hc_for('server', 'ui')
if dom_utils.get_int_number_from_text(
server_data['Memory Available']) < 384:
logger.warning(
'status: RED due to very low memory available on Server')
state.set_error()
elif dom_utils.get_int_number_from_text(
server_data['Memory Available']) < 512:
logger.warning(
'status: ORANGE due to low memory available on Server')
state.set_warn()
if dom_utils.get_int_number_from_text(server_data['Disk Free']) < 256:
logger.warning(
'status: RED due to very low disk free space on Server')
state.set_error()
elif dom_utils.get_int_number_from_text(
server_data['Disk Free']) < 1024:
logger.warning('status: RED due to low disk free space on Server')
state.set_warn()
color_red, color_green, color_blue = delight_utils.get_state_colour(state)
update_blink(state.state)
unicornhathd.set_pixel(ui_utils.to_x(9), 1, purple_r, purple_g, purple_b)
unicornhathd.set_pixel(ui_utils.to_x(11), 1, purple_r, purple_g, purple_b)
unicornhathd.set_pixel(ui_utils.to_x(9), 2, purple_r, purple_g, purple_b)
set_status_for_device(9, 13, color_red, color_green, color_blue)
logger.info('Server: {}'.format(state.get_status_as_light_colour()))
