def test_service_checker_check_by_monit(mock_run):
return_value = 'Monit 5.20.0 uptime: 3h 54m\n' \
'Service Name Status Type\n' \
'sonic Running System\n' \
'sonic1 Not running System\n' \
'telemetry Does not exist Process\n' \
'orchagent Running Process\n' \
'root-overlay Accessible Filesystem\n' \
'var-log Is not accessible Filesystem\n'
mock_run.side_effect = ['active', return_value]
checker = ServiceChecker()
config = Config()
checker.check(config)
assert 'sonic' in checker._info
assert checker._info['sonic'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'sonic1' in checker._info
assert checker._info['sonic1'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'orchagent' in checker._info
assert checker._info['orchagent'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'telemetry' in checker._info
assert checker._info['telemetry'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'root-overlay' in checker._info
assert checker._info['root-overlay'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'var-log' in checker._info
assert checker._info['var-log'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
def test_service_checker_multi_asic(mock_config_db, mock_docker_client):
mock_db_data = MagicMock()
mock_db_data.get_table = MagicMock()
mock_config_db.return_value = mock_db_data
mock_db_data.get_table.return_value = {
'snmp': {
'state': 'enabled',
'has_global_scope': 'True',
'has_per_asic_scope': 'True',
}
}
mock_containers = MagicMock()
mock_snmp_container = MagicMock()
mock_snmp_container.name = 'snmp'
list_return_value = [mock_snmp_container]
for i in range(3):
mock_container = MagicMock()
mock_container.name = 'snmp' + str(i)
list_return_value.append(mock_container)
mock_containers.list = MagicMock(return_value=list_return_value)
mock_docker_client_object = MagicMock()
mock_docker_client.return_value = mock_docker_client_object
mock_docker_client_object.containers = mock_containers
checker = ServiceChecker()
config = Config()
checker.check(config)
assert 'snmp' in checker.container_critical_processes
assert 'snmp:snmpd' in checker._info
assert checker._info['snmp:snmpd'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'snmp0:snmpd' in checker._info
assert checker._info['snmp0:snmpd'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'snmp1:snmpd' in checker._info
assert checker._info['snmp1:snmpd'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'snmp2:snmpd' in checker._info
assert checker._info['snmp2:snmpd'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'snmp:snmp-subagent' in checker._info
assert checker._info['snmp:snmp-subagent'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'snmp0:snmp-subagent' in checker._info
assert checker._info['snmp0:snmp-subagent'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'snmp1:snmp-subagent' in checker._info
assert checker._info['snmp1:snmp-subagent'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'snmp2:snmp-subagent' in checker._info
assert checker._info['snmp2:snmp-subagent'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
def test_service_checker():
return_value = ''
def mock_run_command(cmd):
if cmd == ServiceChecker.CHECK_MONIT_SERVICE_CMD:
return 'active'
else:
return return_value
utils.run_command = mock_run_command
return_value = 'Monit 5.20.0 uptime: 3h 54m\n' \
'Service Name Status Type\n' \
'sonic Running System\n' \
'sonic1 Not running System\n' \
'telemetry Does not exist Process\n' \
'orchagent Running Process\n' \
'root-overlay Accessible Filesystem\n' \
'var-log Is not accessible Filesystem\n'
checker = ServiceChecker()
config = Config()
checker.check(config)
assert 'sonic' in checker._info
assert checker._info['sonic'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'sonic1' in checker._info
assert checker._info['sonic1'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'orchagent' in checker._info
assert checker._info['orchagent'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'telemetry' in checker._info
assert checker._info['telemetry'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'root-overlay' in checker._info
assert checker._info['root-overlay'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'var-log' in checker._info
assert checker._info['var-log'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
def test_service_checker_no_critical_process(mock_get_table, mock_docker_client):
mock_get_table.return_value = {
'snmp': {
'state': 'enabled',
'has_global_scope': 'True',
'has_per_asic_scope': 'True',
}
}
mock_containers = MagicMock()
mock_containers.list = MagicMock(return_value=[])
mock_docker_client_object = MagicMock()
mock_docker_client.return_value = mock_docker_client_object
mock_docker_client_object.containers = mock_containers
checker = ServiceChecker()
config = Config()
checker.check(config)
assert 'system' in checker._info
assert checker._info['system'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
def test_service_checker_single_asic(mock_config_db, mock_run,
mock_docker_client):
mock_db_data = MagicMock()
mock_get_table = MagicMock()
mock_db_data.get_table = mock_get_table
mock_config_db.return_value = mock_db_data
mock_get_table.return_value = {
'snmp': {
'state': 'enabled',
'has_global_scope': 'True',
'has_per_asic_scope': 'False',
}
}
mock_containers = MagicMock()
mock_snmp_container = MagicMock()
mock_snmp_container.name = 'snmp'
mock_containers.list = MagicMock(return_value=[mock_snmp_container])
mock_docker_client_object = MagicMock()
mock_docker_client.return_value = mock_docker_client_object
mock_docker_client_object.containers = mock_containers
mock_run.return_value = mock_supervisorctl_output
checker = ServiceChecker()
assert checker.get_category() == 'Services'
config = Config()
checker.check(config)
assert 'snmp:snmpd' in checker._info
assert checker._info['snmp:snmpd'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'snmp:snmp-subagent' in checker._info
assert checker._info['snmp:snmp-subagent'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
mock_get_table.return_value = {
'new_service': {
'state': 'enabled',
'has_global_scope': 'True',
'has_per_asic_scope': 'False',
},
'snmp': {
'state': 'enabled',
'has_global_scope': 'True',
'has_per_asic_scope': 'False',
}
}
mock_ns_container = MagicMock()
mock_ns_container.name = 'new_service'
mock_containers.list = MagicMock(
return_value=[mock_snmp_container, mock_ns_container])
checker.check(config)
assert 'new_service' in checker.container_critical_processes
assert 'new_service:snmpd' in checker._info
assert checker._info['new_service:snmpd'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'new_service:snmp-subagent' in checker._info
assert checker._info['new_service:snmp-subagent'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
mock_containers.list = MagicMock(return_value=[mock_snmp_container])
checker.check(config)
assert 'new_service' in checker._info
assert checker._info['new_service'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
mock_containers.list = MagicMock(
return_value=[mock_snmp_container, mock_ns_container])
mock_run.return_value = None
checker.check(config)
assert 'new_service:snmpd' in checker._info
assert checker._info['new_service:snmpd'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'new_service:snmp-subagent' in checker._info
assert checker._info['new_service:snmp-subagent'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
origin_container_critical_processes = copy.deepcopy(
checker.container_critical_processes)
checker.save_critical_process_cache()
checker.load_critical_process_cache()
assert origin_container_critical_processes == checker.container_critical_processes
def test_config():
config = Config()
config._config_file = os.path.join(test_path, Config.CONFIG_FILE)
assert config.config_file_exists()
config.load_config()
assert config.interval == 60
assert 'dummy_service' in config.ignore_services
assert 'psu.voltage' in config.ignore_devices
assert len(config.user_defined_checkers) == 0
assert config.get_led_color('fault') == 'orange'
assert config.get_led_color('normal') == 'green'
assert config.get_led_color('booting') == 'orange_blink'
assert config.get_bootup_timeout() == 300
config._reset()
assert not config.ignore_services
assert not config.ignore_devices
assert not config.user_defined_checkers
assert not config.config_data
assert config.get_led_color('fault') == 'red'
assert config.get_led_color('normal') == 'green'
assert config.get_led_color('booting') == 'orange_blink'
config._last_mtime = 1
config._config_file = 'notExistFile'
config.load_config()
assert not config._last_mtime
def test_hardware_checker():
MockConnector.data.update({
'TEMPERATURE_INFO|ASIC': {
'temperature': '20',
'high_threshold': '21'
}
})
MockConnector.data.update({
'FAN_INFO|fan1': {
'presence': 'True',
'status': 'True',
'speed': '60',
'speed_target': '60',
'speed_tolerance': '20'
},
'FAN_INFO|fan2': {
'presence': 'False',
'status': 'True',
'speed': '60',
'speed_target': '60',
'speed_tolerance': '20'
},
'FAN_INFO|fan3': {
'presence': 'True',
'status': 'False',
'speed': '60',
'speed_target': '60',
'speed_tolerance': '20'
},
'FAN_INFO|fan4': {
'presence': 'True',
'status': 'True',
'speed': '20',
'speed_target': '60',
'speed_tolerance': '20'
}
})
MockConnector.data.update({
'PSU_INFO|PSU 1': {
'presence': 'True',
'status': 'True',
'temp': '55',
'temp_threshold': '100',
'voltage': '10',
'voltage_min_threshold': '8',
'voltage_max_threshold': '15',
},
'PSU_INFO|PSU 2': {
'presence': 'False',
'status': 'True',
'temp': '55',
'temp_threshold': '100',
'voltage': '10',
'voltage_min_threshold': '8',
'voltage_max_threshold': '15',
},
'PSU_INFO|PSU 3': {
'presence': 'True',
'status': 'False',
'temp': '55',
'temp_threshold': '100',
'voltage': '10',
'voltage_min_threshold': '8',
'voltage_max_threshold': '15',
},
'PSU_INFO|PSU 4': {
'presence': 'True',
'status': 'True',
'temp': '101',
'temp_threshold': '100',
'voltage': '10',
'voltage_min_threshold': '8',
'voltage_max_threshold': '15',
},
'PSU_INFO|PSU 5': {
'presence': 'True',
'status': 'True',
'temp': '55',
'temp_threshold': '100',
'voltage': '10',
'voltage_min_threshold': '12',
'voltage_max_threshold': '15',
}
})
checker = HardwareChecker()
assert checker.get_category() == 'Hardware'
config = Config()
checker.check(config)
assert 'ASIC' in checker._info
assert checker._info['ASIC'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'fan1' in checker._info
assert checker._info['fan1'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'fan2' in checker._info
assert checker._info['fan2'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'fan3' in checker._info
assert checker._info['fan3'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'fan4' in checker._info
assert checker._info['fan4'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'PSU 1' in checker._info
assert checker._info['PSU 1'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
assert 'PSU 2' in checker._info
assert checker._info['PSU 2'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'PSU 3' in checker._info
assert checker._info['PSU 3'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'PSU 4' in checker._info
assert checker._info['PSU 4'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
assert 'PSU 5' in checker._info
assert checker._info['PSU 5'][
HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
