def test__ensureService_performs_stop_for_on_service(self):
service = make_fake_service(SERVICE_STATE.OFF)
service_monitor = self.make_service_monitor([service])
mock_getServiceState = self.patch(
service_monitor, "getServiceState")
mock_getServiceState.side_effect = [
succeed(ServiceState(SERVICE_STATE.ON, "running")),
succeed(ServiceState(SERVICE_STATE.OFF, "waiting")),
]
mock_performServiceAction = self.patch(
service_monitor, "_performServiceAction")
mock_performServiceAction.return_value = succeed(None)
with FakeLogger(
"maas.service_monitor", level=logging.INFO) as maaslog:
yield service_monitor._ensureService(service)
self.assertThat(
mock_performServiceAction, MockCalledOnceWith(service, "stop"))
self.assertDocTestMatches(
"""\
Service '%s' is not off, it will be stopped.
Service '%s' has been stopped and is 'waiting'.
""" % (service.service_name, service.service_name),
maaslog.output)
def test__ensureService_performs_raises_ServiceActionError(self):
service = make_fake_service(SERVICE_STATE.ON)
service_monitor = self.make_service_monitor([service])
mock_getServiceState = self.patch(service_monitor, "getServiceState")
mock_getServiceState.side_effect = [
succeed(ServiceState(SERVICE_STATE.OFF, "waiting")),
succeed(ServiceState(SERVICE_STATE.OFF, "waiting")),
]
mock_performServiceAction = self.patch(service_monitor,
"_performServiceAction")
mock_performServiceAction.return_value = succeed(None)
with ExpectedException(ServiceActionError):
with FakeLogger("maas.service_monitor",
level=logging.INFO) as maaslog:
yield service_monitor._ensureService(service)
self.assertDocTestMatches(
"""\
Service '%s' is not on, it will be started.
Service '%s' failed to start. Its current state is '%s' and '%s'.
""" % (
service.service_name,
service.service_name,
SERVICE_STATE.OFF.value,
"waiting",
), maaslog.output)
def test__returns_service_status_string(self):
# Make sure the short status string makes sense.
service = make_fake_service(self.state_expected)
state = ServiceState(self.state_observed, None)
status_string, status_message = yield state.getStatusInfo(service)
self.assertThat(status_string, Equals(
self.expected_status_strings[
self.state_observed, self.state_expected]))
def test_updates_services_in_database(self):
# Pretend we're in a production environment.
self.patch(service_monitor_service,
"is_dev_environment").return_value = False
self.patch(proxyconfig, "is_config_present").return_value = True
service = self.pick_service()
state = ServiceState(SERVICE_STATE.ON, "running")
mock_ensureServices = self.patch(service_monitor, "ensureServices")
mock_ensureServices.return_value = succeed({service.name: state})
region = yield deferToDatabase(
transactional(factory.make_RegionController))
self.patch(RegionController.objects,
"get_running_controller").return_value = region
monitor_service = ServiceMonitorService(Clock())
yield monitor_service.startService()
yield monitor_service.stopService()
service = yield deferToDatabase(transactional(Service.objects.get),
node=region,
name=service.name)
self.assertThat(
service,
MatchesStructure.byEquality(
name=service.name,
status=SERVICE_STATUS.RUNNING,
status_info="",
),
)
def test__writes_config_and_doesnt_use_omapi_when_was_off(self):
write_file = self.patch_sudo_write_file()
get_service_state = self.patch_getServiceState()
get_service_state.return_value = ServiceState(
SERVICE_STATE.OFF, "dead")
restart_service = self.patch_restartService()
ensure_service = self.patch_ensureService()
update_hosts = self.patch_update_hosts()
failover_peers = make_failover_peer_config()
shared_network = make_shared_network()
[shared_network] = fix_shared_networks_failover(
[shared_network], [failover_peers])
host = make_host(dhcp_snippets=[])
interface = make_interface()
global_dhcp_snippets = make_global_dhcp_snippets()
expected_config = factory.make_name('config')
self.patch_get_config().return_value = expected_config
dhcp_service = dhcp.service_monitor.getServiceByName(
self.server.dhcp_service)
on = self.patch_autospec(dhcp_service, "on")
omapi_key = factory.make_name('omapi_key')
old_host = make_host(dhcp_snippets=[])
old_state = dhcp.DHCPState(
omapi_key, [failover_peers], [shared_network],
[old_host], [interface], global_dhcp_snippets)
dhcp._current_server_state[self.server.dhcp_service] = old_state
yield self.configure(
omapi_key,
[failover_peers], [shared_network], [host], [interface],
global_dhcp_snippets)
self.assertThat(
write_file,
MockCallsMatch(
call(
self.server.config_filename,
expected_config.encode("utf-8")),
call(
self.server.interfaces_filename,
interface["name"].encode("utf-8")),
))
self.assertThat(on, MockCalledOnceWith())
self.assertThat(
get_service_state,
MockCalledOnceWith(self.server.dhcp_service, now=True))
self.assertThat(
restart_service, MockNotCalled())
self.assertThat(
ensure_service, MockCalledOnceWith(self.server.dhcp_service))
self.assertThat(
update_hosts, MockNotCalled())
self.assertEquals(
dhcp._current_server_state[self.server.dhcp_service],
dhcp.DHCPState(
omapi_key, [failover_peers], [shared_network],
[host], [interface], global_dhcp_snippets))
def test__reloadService_always_raises_error_if_fails_to_start(self):
fake_service = make_fake_service(SERVICE_STATE.ON)
service_monitor = self.make_service_monitor([fake_service])
mock_ensureService = self.patch(service_monitor, "ensureService")
mock_ensureService.return_value = succeed(
ServiceState(SERVICE_STATE.OFF, "dead"))
with ExpectedException(ServiceActionError):
yield service_monitor.reloadService(fake_service.name, if_on=True)
def test__ensureService_does_nothing_when_any_state_expected(self):
service = make_fake_service(SERVICE_STATE.ANY)
service_monitor = self.make_service_monitor([service])
self.patch_autospec(service_monitor, "getServiceState")
self.patch_autospec(service_monitor, "_performServiceAction")
self.assertThat((yield service_monitor._ensureService(service)),
Equals(ServiceState(SERVICE_STATE.UNKNOWN)))
self.assertThat(service_monitor.getServiceState, MockNotCalled())
self.assertThat(service_monitor._performServiceAction, MockNotCalled())
def test__ensureServices_calls__ensureService(self):
fake_service = make_fake_service()
service_monitor = self.make_service_monitor([fake_service])
active_state = pick_observed_state()
process_state = random.choice(["running", "dead"])
service_state = ServiceState(active_state, process_state)
mock_ensureService = self.patch(service_monitor, "_ensureService")
mock_ensureService.return_value = succeed(service_state)
observed = yield service_monitor.ensureService(fake_service.name)
self.assertEquals(service_state, observed)
self.assertThat(mock_ensureService, MockCalledOnceWith(fake_service))
def test__restartService_raises_ServiceActionError_if_not_on(self):
fake_service = make_fake_service(SERVICE_STATE.ON)
service_monitor = self.make_service_monitor([fake_service])
mock_performServiceAction = self.patch(service_monitor,
"_performServiceAction")
mock_performServiceAction.return_value = succeed(None)
active_state = pick_observed_state(but_not={SERVICE_STATE.ON})
service_state = ServiceState(active_state, "dead")
mock_getServiceState = self.patch(service_monitor, "getServiceState")
mock_getServiceState.return_value = succeed(service_state)
with ExpectedException(ServiceActionError):
yield service_monitor.restartService(fake_service.name)
def test_reports_services_to_region_when_changed(self):
# Pretend we're in a production environment.
self.patch(sms, "is_dev_environment").return_value = False
protocol, connecting = self.patch_rpc_methods()
self.addCleanup((yield connecting))
class ExampleService(AlwaysOnService):
name = service_name = snap_service_name = (
factory.make_name("service"))
service = ExampleService()
# Inveigle this new service into the service monitor.
self.addCleanup(service_monitor._services.pop, service.name)
service_monitor._services[service.name] = service
state = ServiceState(SERVICE_STATE.ON, "running")
mock_ensureServices = self.patch(service_monitor, "ensureServices")
mock_ensureServices.return_value = succeed({
service.name: state
})
client = getRegionClient()
rpc_service = Mock()
rpc_service.getClientNow.return_value = succeed(client)
monitor_service = sms.ServiceMonitorService(
rpc_service, Clock())
monitor_service._services = yield monitor_service._buildServices({
service.name: state
})
# Force last reported state to dead. That way an update is performed.
orig_cached_services = monitor_service._services
for ser in orig_cached_services:
ser['status'] = 'dead'
yield monitor_service.startService()
yield monitor_service.stopService()
expected_services = list(monitor_service.ALWAYS_RUNNING_SERVICES)
expected_services.append({
"name": service.name,
"status": "running",
"status_info": "",
})
self.assertThat(
protocol.UpdateServices,
MockCalledOnceWith(
protocol,
system_id=client.localIdent,
services=expected_services))
self.assertIsNot(orig_cached_services, monitor_service._services)
def test_buildServices_builds_services_list(self):
self.patch(proxyconfig, "is_config_present").return_value = True
monitor_service = ServiceMonitorService(Clock())
service = self.pick_service()
state = ServiceState(SERVICE_STATE.ON, "running")
observed_services = yield monitor_service._buildServices(
{service.name: state})
expected_services = [{
"name": service.name,
"status": "running",
"status_info": ""
}]
self.assertEqual(expected_services, observed_services)
def test__ensureServices_returns_dict_for_states(self):
fake_services = [make_fake_service() for _ in range(3)]
expected_states = {}
for service in fake_services:
active_state = pick_observed_state()
process_state = random.choice(["running", "dead"])
expected_states[service.name] = ServiceState(
active_state, process_state)
service_monitor = self.make_service_monitor(fake_services)
self.patch(service_monitor, "ensureService").side_effect = (
lambda name: succeed(expected_states[name]))
observed = yield service_monitor.ensureServices()
self.assertEquals(expected_states, observed)
def test__returns_service_info_message(self):
# Make sure any message given by a service gets passed through, except
# when the service is dead or off and expected to be on, in which case
# a message is manufactured.
example_message = factory.make_string(60, spaces=True)
example_process_state = factory.make_name("process-state")
service = make_fake_service(self.state_expected, example_message)
state = ServiceState(self.state_observed, example_process_state)
status_string, status_message = yield state.getStatusInfo(service)
if self.state_expected == SERVICE_STATE.ON:
if self.state_observed == SERVICE_STATE.OFF:
self.assertThat(status_message, Equals(
service.service_name + " is currently stopped."))
elif self.state_observed == SERVICE_STATE.DEAD:
self.assertThat(status_message, Equals(
service.service_name + " failed to start, process "
"result: (" + example_process_state + ")"))
else:
self.assertThat(status_message, Equals(example_message))
else:
self.assertThat(status_message, Equals(example_message))
def test__reloadService_always_calls_ensureService_then_reloads(self):
fake_service = make_fake_service(SERVICE_STATE.ON)
service_monitor = self.make_service_monitor([fake_service])
mock_performServiceAction = self.patch(service_monitor,
"_performServiceAction")
mock_performServiceAction.return_value = succeed(None)
mock_ensureService = self.patch(service_monitor, "ensureService")
mock_ensureService.return_value = succeed(
ServiceState(SERVICE_STATE.ON, "running"))
yield service_monitor.reloadService(fake_service.name, if_on=True)
self.assertThat(mock_ensureService,
MockCalledOnceWith(fake_service.name))
self.assertThat(mock_performServiceAction,
MockCalledOnceWith(fake_service, "reload"))
def test_buildServices_adds_services_to_always_running_services(self):
monitor_service = sms.ServiceMonitorService(sentinel.client_service,
Clock())
service = self.pick_service()
state = ServiceState(SERVICE_STATE.ON, "running")
observed_services = yield monitor_service._buildServices(
{service.name: state})
expected_services = list(monitor_service.ALWAYS_RUNNING_SERVICES)
expected_services.append({
"name": service.name,
"status": "running",
"status_info": ""
})
self.assertEquals(expected_services, observed_services)
def test__restartService_performs_restart(self):
fake_service = make_fake_service(SERVICE_STATE.ON)
service_monitor = self.make_service_monitor([fake_service])
mock_performServiceAction = self.patch(service_monitor,
"_performServiceAction")
mock_performServiceAction.return_value = succeed(None)
service_state = ServiceState(SERVICE_STATE.ON, "running")
mock_getServiceState = self.patch(service_monitor, "getServiceState")
mock_getServiceState.return_value = succeed(service_state)
observed = yield service_monitor.restartService(fake_service.name)
self.assertEquals(service_state, observed)
self.assertThat(mock_getServiceState,
MockCalledOnceWith(fake_service.name, now=True))
self.assertThat(mock_performServiceAction,
MockCalledOnceWith(fake_service, "restart"))
def test___ensureService_allows_dead_for_off_service(self):
service = make_fake_service(SERVICE_STATE.OFF)
service_monitor = self.make_service_monitor([service])
mock_getServiceState = self.patch(service_monitor, "getServiceState")
mock_getServiceState.return_value = succeed(
ServiceState(SERVICE_STATE.DEAD, "Result: exit-code"))
with FakeLogger("maas.service_monitor",
level=logging.DEBUG) as maaslog:
yield service_monitor._ensureService(service)
self.assertDocTestMatches(
"Service '%s' is %s and '%s'." %
(service.service_name, SERVICE_STATE.DEAD, "Result: exit-code"),
maaslog.output)
def test___ensureService_allows_dead_for_off_service(self):
service = make_fake_service(SERVICE_STATE.OFF)
service_monitor = self.make_service_monitor([service])
mock_getServiceState = self.patch(
service_monitor, "getServiceState")
mock_getServiceState.return_value = succeed(
ServiceState(SERVICE_STATE.DEAD, "Result: exit-code"))
log = self.patch(service_monitor_module, 'log')
yield service_monitor._ensureService(service)
self.assertThat(log.debug, MockCalledOnceWith(
"Service '{name}' is {state} and '{process}'.",
name=service.service_name, state=SERVICE_STATE.DEAD,
process="Result: exit-code"))
def test___ensureService_logs_debug_in_expected_states(self):
state = SERVICE_STATE.ON
service = make_fake_service(state)
service_monitor = self.make_service_monitor([service])
expected_process_state = service_monitor.PROCESS_STATE[state]
mock_getServiceState = self.patch(
service_monitor, "getServiceState")
mock_getServiceState.return_value = succeed(
ServiceState(SERVICE_STATE.ON, expected_process_state))
log = self.patch(service_monitor_module, 'log')
yield service_monitor._ensureService(service)
self.assertThat(log.debug, MockCalledOnceWith(
"Service '{name}' is {state} and '{process}'.",
name=service.service_name, state=state,
process=expected_process_state))
def test___ensureService_logs_debug_in_expected_states(self):
state = SERVICE_STATE.ON
service = make_fake_service(state)
service_monitor = self.make_service_monitor([service])
expected_process_state = service_monitor.PROCESS_STATE[state]
mock_getServiceState = self.patch(service_monitor, "getServiceState")
mock_getServiceState.return_value = succeed(
ServiceState(SERVICE_STATE.ON, expected_process_state))
with FakeLogger("maas.service_monitor",
level=logging.DEBUG) as maaslog:
yield service_monitor._ensureService(service)
self.assertDocTestMatches(
"Service '%s' is %s and '%s'." %
(service.service_name, state, expected_process_state),
maaslog.output)
def test___ensureService_logs_mismatch_for_dead_process_state(self):
service = make_fake_service(SERVICE_STATE.OFF)
service_monitor = self.make_service_monitor([service])
invalid_process_state = factory.make_name("invalid")
mock_getServiceState = self.patch(service_monitor, "getServiceState")
mock_getServiceState.return_value = succeed(
ServiceState(SERVICE_STATE.DEAD, invalid_process_state))
with FakeLogger("maas.service_monitor",
level=logging.WARNING) as maaslog:
yield service_monitor._ensureService(service)
self.assertDocTestMatches(
"Service '%s' is %s but not in the expected state of "
"'%s', its current state is '%s'." %
(service.service_name, SERVICE_STATE.DEAD.value,
service_monitor.PROCESS_STATE[SERVICE_STATE.DEAD],
invalid_process_state), maaslog.output)
def test_doesnt_reports_services_to_region_when_the_same_status(self):
# Pretend we're in a production environment.
self.patch(sms, "is_dev_environment").return_value = False
protocol, connecting = self.patch_rpc_methods()
self.addCleanup((yield connecting))
class ExampleService(AlwaysOnService):
name = service_name = snap_service_name = (
factory.make_name("service"))
service = ExampleService()
# Inveigle this new service into the service monitor.
self.addCleanup(service_monitor._services.pop, service.name)
service_monitor._services[service.name] = service
state = ServiceState(SERVICE_STATE.ON, "running")
mock_ensureServices = self.patch(service_monitor, "ensureServices")
mock_ensureServices.return_value = succeed({
service.name: state
})
client = getRegionClient()
rpc_service = Mock()
rpc_service.getClientNow.return_value = succeed(client)
monitor_service = sms.ServiceMonitorService(
rpc_service, Clock())
monitor_service._services = yield monitor_service._buildServices({
service.name: state
})
yield monitor_service.startService()
yield monitor_service.stopService()
self.assertThat(
protocol.UpdateServices,
MockNotCalled())
def test__ensureServices_handles_errors(self):
services = make_fake_service(), make_fake_service()
service_monitor = self.make_service_monitor(services)
# Plant some states into the monitor's memory.
service_states = {
service.name: ServiceState(pick_observed_state(),
random.choice(["running", "dead"]))
for service in services
}
service_monitor._serviceStates.update(service_states)
# Make both service monitor checks fail with a distinct error.
self.patch(service_monitor, "ensureService")
def raise_exception(service_name):
raise factory.make_exception(service_name + " broke")
def raise_exception_later(service_name):
# We use deferLater() to ensure that `raise_exception` is called
# asynchronously; this helps to ensure that ensureServices() has
# not closed over mutating local state, e.g. a loop variable.
return deferLater(reactor, 0, raise_exception, service_name)
service_monitor.ensureService.side_effect = raise_exception_later
# Capture logs when calling ensureServices().
with FakeLogger("maas.service_monitor") as logger:
observed = yield service_monitor.ensureServices()
# The errors mean we were returned the states planted earlier.
self.assertThat(observed, Equals(service_states))
# The errors were logged with the service name and message.
for service in services:
self.assertThat(
logger.output,
Contains(
"While monitoring service '%s' an error was encountered: "
"%s broke" % (service.name, service.name)))
def test_writes_config_and_restarts_when_omapi_fails(self):
write_file = self.patch_sudo_write_file()
get_service_state = self.patch_getServiceState()
get_service_state.return_value = ServiceState(SERVICE_STATE.ON,
"running")
restart_service = self.patch_restartService()
ensure_service = self.patch_ensureService()
update_hosts = self.patch_update_hosts()
update_hosts.side_effect = factory.make_exception()
failover_peers = make_failover_peer_config()
shared_network = make_shared_network()
[shared_network] = fix_shared_networks_failover([shared_network],
[failover_peers])
old_hosts = [make_host(dhcp_snippets=[]) for _ in range(3)]
interface = make_interface()
global_dhcp_snippets = make_global_dhcp_snippets()
expected_config = factory.make_name("config")
self.patch_get_config().return_value = expected_config
dhcp_service = dhcp.service_monitor.getServiceByName(
self.server.dhcp_service)
on = self.patch_autospec(dhcp_service, "on")
omapi_key = factory.make_name("omapi_key")
old_state = dhcp.DHCPState(
omapi_key,
[failover_peers],
[shared_network],
old_hosts,
[interface],
global_dhcp_snippets,
)
dhcp._current_server_state[self.server.dhcp_service] = old_state
new_hosts = copy.deepcopy(old_hosts)
removed_host = new_hosts.pop()
modified_host = new_hosts[0]
modified_host["ip"] = factory.make_ip_address(
ipv6=self.server.dhcp_service == "DHCPv6")
added_host = make_host(dhcp_snippets=[])
new_hosts.append(added_host)
with FakeLogger("maas") as logger:
yield self.configure(
omapi_key,
[failover_peers],
[shared_network],
new_hosts,
[interface],
global_dhcp_snippets,
)
self.assertThat(
write_file,
MockCallsMatch(
call(
self.server.config_filename,
expected_config.encode("utf-8"),
mode=0o640,
),
call(
self.server.interfaces_filename,
interface["name"].encode("utf-8"),
mode=0o640,
),
),
)
self.assertThat(on, MockCalledOnceWith())
self.assertThat(
get_service_state,
MockCalledOnceWith(self.server.dhcp_service, now=True),
)
self.assertThat(restart_service,
MockCalledOnceWith(self.server.dhcp_service))
self.assertThat(ensure_service,
MockCalledOnceWith(self.server.dhcp_service))
self.assertThat(
update_hosts,
MockCalledOnceWith(ANY, [removed_host], [added_host],
[modified_host]),
)
self.assertEqual(
dhcp._current_server_state[self.server.dhcp_service],
dhcp.DHCPState(
omapi_key,
[failover_peers],
[shared_network],
new_hosts,
[interface],
global_dhcp_snippets,
),
)
self.assertDocTestMatches(
"Failed to update all host maps. Restarting DHCPv... "
"service to ensure host maps are in-sync.",
logger.output,
)
def test__writes_config_and_uses_omapi_to_update_hosts(self):
write_file = self.patch_sudo_write_file()
get_service_state = self.patch_getServiceState()
get_service_state.return_value = ServiceState(SERVICE_STATE.ON,
"running")
restart_service = self.patch_restartService()
ensure_service = self.patch_ensureService()
update_hosts = self.patch_update_hosts()
failover_peers = make_failover_peer_config()
shared_network = make_shared_network()
[shared_network] = fix_shared_networks_failover([shared_network],
[failover_peers])
old_hosts = [make_host(dhcp_snippets=[]) for _ in range(3)]
interface = make_interface()
global_dhcp_snippets = make_global_dhcp_snippets()
expected_config = factory.make_name("config")
self.patch_get_config().return_value = expected_config
dhcp_service = dhcp.service_monitor.getServiceByName(
self.server.dhcp_service)
on = self.patch_autospec(dhcp_service, "on")
omapi_key = factory.make_name("omapi_key")
old_state = dhcp.DHCPState(
omapi_key,
[failover_peers],
[shared_network],
old_hosts,
[interface],
global_dhcp_snippets,
)
dhcp._current_server_state[self.server.dhcp_service] = old_state
new_hosts = copy.deepcopy(old_hosts)
removed_host = new_hosts.pop()
modified_host = new_hosts[0]
modified_host["ip"] = factory.make_ip_address()
added_host = make_host(dhcp_snippets=[])
new_hosts.append(added_host)
yield self.configure(
omapi_key,
[failover_peers],
[shared_network],
new_hosts,
[interface],
global_dhcp_snippets,
)
self.assertThat(
write_file,
MockCallsMatch(
call(
self.server.config_filename,
expected_config.encode("utf-8"),
mode=0o640,
),
call(
self.server.interfaces_filename,
interface["name"].encode("utf-8"),
mode=0o640,
),
),
)
self.assertThat(on, MockCalledOnceWith())
self.assertThat(
get_service_state,
MockCalledOnceWith(self.server.dhcp_service, now=True),
)
self.assertThat(restart_service, MockNotCalled())
self.assertThat(ensure_service,
MockCalledOnceWith(self.server.dhcp_service))
self.assertThat(
update_hosts,
MockCalledOnceWith(ANY, [removed_host], [added_host],
[modified_host]),
)
self.assertEquals(
dhcp._current_server_state[self.server.dhcp_service],
dhcp.DHCPState(
omapi_key,
[failover_peers],
[shared_network],
new_hosts,
[interface],
global_dhcp_snippets,
),
)
