def test_non_zero_exit(self) -> None: """ Test non zero service exit multiple times """ self._non_zero_exit_fixture() update_stats(self.serviceName) wrapper = ServiceStateWrapper() status = wrapper.get_service_status(self.serviceName) self.assertEqual(status.latest_rc, 2, "service exit status check") self.assertEqual(status.latest_service_result, ServiceExitStatus.ServiceResult.Value("EXIT_CODE")) self.assertEqual(status.latest_exit_code, ServiceExitStatus.ExitCode.Value("EXITED")) self.assertEqual(status.num_fail_exits, 1) self.assertEqual(status.num_clean_exits, 0) os.environ["EXIT_STATUS"] = "3" update_stats(self.serviceName) status = wrapper.get_service_status(self.serviceName) self.assertEqual(status.latest_rc, 3, "service exit status check") self.assertEqual(status.latest_service_result, ServiceExitStatus.ServiceResult.Value("EXIT_CODE")) self.assertEqual(status.latest_exit_code, ServiceExitStatus.ExitCode.Value("EXITED")) self.assertEqual(status.num_fail_exits, 2) self.assertEqual(status.num_clean_exits, 0)
def update_stats(service: str) -> None: status = get_status() wrapper_obj = ServiceStateWrapper() try: wrapper_obj.update_service_status(service, status) except Exception as e: logging.error('Failed to write to redis, status %s', e) logging.error('Logging exit info instead \n%s', status)
def _collect_service_restart_stats(): """ Collect the success and failure restarts for services """ service_dict = ServiceStateWrapper().get_all_services_status() for service_name, status in service_dict.items(): SERVICE_RESTART_STATUS.labels(service_name=service_name, status="Failure").set( status.num_fail_exits) SERVICE_RESTART_STATUS.labels(service_name=service_name, status="Success").set( status.num_clean_exits)
def _collect_service_restart_stats(): """ Collect the success and failure restarts for services """ try: service_dict = ServiceStateWrapper().get_all_services_status() except Exception as e: logging.error("Could not fetch service status: %s", e) return for service_name, status in service_dict.items(): SERVICE_RESTART_STATUS.labels(service_name=service_name, status="Failure").set( status.num_fail_exits) SERVICE_RESTART_STATUS.labels(service_name=service_name, status="Success").set( status.num_clean_exits)
def main(): """ Top-level function for health service """ service = MagmaService('health', None) # Optionally pipe errors to Sentry sentry_init() # Service state wrapper obj service_state = ServiceStateWrapper() # Load service YML config state_recovery_config = service.config["state_recovery"] services_check = state_recovery_config["services_check"] polling_interval = int(state_recovery_config["interval_check_mins"]) * 60 restart_threshold = state_recovery_config["restart_threshold"] snapshots_dir = state_recovery_config["snapshots_dir"] redis_dump_src = load_service_config("redis").get("dir", "/var/opt/magma") state_recovery_job = StateRecoveryJob(service_state=service_state, polling_interval=polling_interval, services_check=services_check, restart_threshold=restart_threshold, redis_dump_src=redis_dump_src, snapshots_dir=snapshots_dir, service_loop=service.loop) state_recovery_job.start() # Run the service loop service.run() # Cleanup the service service.close()
def test_coredump_exit(self) -> None: """ Test core dump exit and also recovery after a core dump """ self._core_dump_fixture() update_stats(self.serviceName) wrapper = ServiceStateWrapper() status = wrapper.get_service_status(self.serviceName) self.assertEqual(status.latest_rc, 0, "service exit status check") self.assertEqual(status.latest_service_result, ServiceExitStatus.ServiceResult.Value("CORE_DUMP")) self.assertEqual(status.latest_exit_code, ServiceExitStatus.ExitCode.Value("DUMPED")) self.assertEqual(status.num_fail_exits, 1) self.assertEqual(status.num_clean_exits, 0) update_stats(self.serviceName) status = wrapper.get_service_status(self.serviceName) self.assertEqual(status.num_fail_exits, 2) self.assertEqual(status.num_clean_exits, 0) # Test that we can do a clean update after exits self._success_fixture() update_stats(self.serviceName) wrapper = ServiceStateWrapper() status = wrapper.get_service_status(self.serviceName) self.assertEqual(status.latest_rc, 0, "service exit status check") self.assertEqual(status.latest_service_result, ServiceExitStatus.ServiceResult.Value("SUCCESS"), "Service result check") self.assertEqual(status.latest_exit_code, ServiceExitStatus.ExitCode.Value("EXITED"), "Service exit check") self.assertEqual(status.num_clean_exits, 1, "Clean exit check") self.assertEqual(status.num_fail_exits, 2)
def test_success_exit(self) -> None: """ Test successful exits for a service multiple times and for multiple services """ self._success_fixture() update_stats(self.serviceName) wrapper = ServiceStateWrapper() status = wrapper.get_service_status(self.serviceName) self.assertEqual(status.latest_rc, 0, "service exit status check") self.assertEqual(status.latest_service_result, ServiceExitStatus.ServiceResult.Value("SUCCESS"), "Service result check") self.assertEqual(status.latest_exit_code, ServiceExitStatus.ExitCode.Value("EXITED"), "Service exit check") self.assertEqual(status.num_clean_exits, 1, "Clean exit check") # Multiple restarts update_stats(self.serviceName) status = wrapper.get_service_status(self.serviceName) self.assertEqual(status.latest_rc, 0, "service exit status check") self.assertEqual(status.latest_service_result, ServiceExitStatus.ServiceResult.Value("SUCCESS"), "Service result check") self.assertEqual(status.latest_exit_code, ServiceExitStatus.ExitCode.Value("EXITED"), "Service exit check") self.assertEqual(status.num_clean_exits, 2, "Clean exit check") # Multiple service restarts update_stats(self.serviceName2) status = wrapper.get_service_status(self.serviceName2) self.assertEqual(status.latest_rc, 0, "service exit status check") self.assertEqual(status.latest_service_result, ServiceExitStatus.ServiceResult.Value("SUCCESS"), "Service result check") self.assertEqual(status.latest_exit_code, ServiceExitStatus.ExitCode.Value("EXITED"), "Service exit code check") self.assertEqual(status.num_clean_exits, 1, "Clean exit check") self.assertEqual(status.num_fail_exits, 0, "Failure exit status")
def tearDown(self): wrapper = ServiceStateWrapper() wrapper.cleanup_service_status()