def test__ensureService_performs_stop_for_on_service(self): service = make_fake_service(SERVICE_STATE.OFF) service_monitor = self.make_service_monitor([service]) mock_getServiceState = self.patch( service_monitor, "getServiceState") mock_getServiceState.side_effect = [ succeed(ServiceState(SERVICE_STATE.ON, "running")), succeed(ServiceState(SERVICE_STATE.OFF, "waiting")), ] mock_performServiceAction = self.patch( service_monitor, "_performServiceAction") mock_performServiceAction.return_value = succeed(None) with FakeLogger( "maas.service_monitor", level=logging.INFO) as maaslog: yield service_monitor._ensureService(service) self.assertThat( mock_performServiceAction, MockCalledOnceWith(service, "stop")) self.assertDocTestMatches( """\ Service '%s' is not off, it will be stopped. Service '%s' has been stopped and is 'waiting'. """ % (service.service_name, service.service_name), maaslog.output)
def test__ensureService_performs_raises_ServiceActionError(self): service = make_fake_service(SERVICE_STATE.ON) service_monitor = self.make_service_monitor([service]) mock_getServiceState = self.patch(service_monitor, "getServiceState") mock_getServiceState.side_effect = [ succeed(ServiceState(SERVICE_STATE.OFF, "waiting")), succeed(ServiceState(SERVICE_STATE.OFF, "waiting")), ] mock_performServiceAction = self.patch(service_monitor, "_performServiceAction") mock_performServiceAction.return_value = succeed(None) with ExpectedException(ServiceActionError): with FakeLogger("maas.service_monitor", level=logging.INFO) as maaslog: yield service_monitor._ensureService(service) self.assertDocTestMatches( """\ Service '%s' is not on, it will be started. Service '%s' failed to start. Its current state is '%s' and '%s'. """ % ( service.service_name, service.service_name, SERVICE_STATE.OFF.value, "waiting", ), maaslog.output)
def test_updates_services_in_database(self): # Pretend we're in a production environment. self.patch(service_monitor_service, "is_dev_environment").return_value = False self.patch(proxyconfig, "is_config_present").return_value = True service = self.pick_service() state = ServiceState(SERVICE_STATE.ON, "running") mock_ensureServices = self.patch(service_monitor, "ensureServices") mock_ensureServices.return_value = succeed({service.name: state}) region = yield deferToDatabase( transactional(factory.make_RegionController)) self.patch(RegionController.objects, "get_running_controller").return_value = region monitor_service = ServiceMonitorService(Clock()) yield monitor_service.startService() yield monitor_service.stopService() service = yield deferToDatabase(transactional(Service.objects.get), node=region, name=service.name) self.assertThat( service, MatchesStructure.byEquality( name=service.name, status=SERVICE_STATUS.RUNNING, status_info="", ), )
def test__writes_config_and_doesnt_use_omapi_when_was_off(self): write_file = self.patch_sudo_write_file() get_service_state = self.patch_getServiceState() get_service_state.return_value = ServiceState( SERVICE_STATE.OFF, "dead") restart_service = self.patch_restartService() ensure_service = self.patch_ensureService() update_hosts = self.patch_update_hosts() failover_peers = make_failover_peer_config() shared_network = make_shared_network() [shared_network] = fix_shared_networks_failover( [shared_network], [failover_peers]) host = make_host(dhcp_snippets=[]) interface = make_interface() global_dhcp_snippets = make_global_dhcp_snippets() expected_config = factory.make_name('config') self.patch_get_config().return_value = expected_config dhcp_service = dhcp.service_monitor.getServiceByName( self.server.dhcp_service) on = self.patch_autospec(dhcp_service, "on") omapi_key = factory.make_name('omapi_key') old_host = make_host(dhcp_snippets=[]) old_state = dhcp.DHCPState( omapi_key, [failover_peers], [shared_network], [old_host], [interface], global_dhcp_snippets) dhcp._current_server_state[self.server.dhcp_service] = old_state yield self.configure( omapi_key, [failover_peers], [shared_network], [host], [interface], global_dhcp_snippets) self.assertThat( write_file, MockCallsMatch( call( self.server.config_filename, expected_config.encode("utf-8")), call( self.server.interfaces_filename, interface["name"].encode("utf-8")), )) self.assertThat(on, MockCalledOnceWith()) self.assertThat( get_service_state, MockCalledOnceWith(self.server.dhcp_service, now=True)) self.assertThat( restart_service, MockNotCalled()) self.assertThat( ensure_service, MockCalledOnceWith(self.server.dhcp_service)) self.assertThat( update_hosts, MockNotCalled()) self.assertEquals( dhcp._current_server_state[self.server.dhcp_service], dhcp.DHCPState( omapi_key, [failover_peers], [shared_network], [host], [interface], global_dhcp_snippets))
def test__reloadService_always_raises_error_if_fails_to_start(self): fake_service = make_fake_service(SERVICE_STATE.ON) service_monitor = self.make_service_monitor([fake_service]) mock_ensureService = self.patch(service_monitor, "ensureService") mock_ensureService.return_value = succeed( ServiceState(SERVICE_STATE.OFF, "dead")) with ExpectedException(ServiceActionError): yield service_monitor.reloadService(fake_service.name, if_on=True)
def test__returns_service_status_string(self): # Make sure the short status string makes sense. service = make_fake_service(self.state_expected) state = ServiceState(self.state_observed, None) status_string, status_message = yield state.getStatusInfo(service) self.assertThat(status_string, Equals( self.expected_status_strings[ self.state_observed, self.state_expected]))
def test__ensureService_does_nothing_when_any_state_expected(self): service = make_fake_service(SERVICE_STATE.ANY) service_monitor = self.make_service_monitor([service]) self.patch_autospec(service_monitor, "getServiceState") self.patch_autospec(service_monitor, "_performServiceAction") self.assertThat((yield service_monitor._ensureService(service)), Equals(ServiceState(SERVICE_STATE.UNKNOWN))) self.assertThat(service_monitor.getServiceState, MockNotCalled()) self.assertThat(service_monitor._performServiceAction, MockNotCalled())
def test__ensureServices_calls__ensureService(self): fake_service = make_fake_service() service_monitor = self.make_service_monitor([fake_service]) active_state = pick_observed_state() process_state = random.choice(["running", "dead"]) service_state = ServiceState(active_state, process_state) mock_ensureService = self.patch(service_monitor, "_ensureService") mock_ensureService.return_value = succeed(service_state) observed = yield service_monitor.ensureService(fake_service.name) self.assertEquals(service_state, observed) self.assertThat(mock_ensureService, MockCalledOnceWith(fake_service))
def test__restartService_raises_ServiceActionError_if_not_on(self): fake_service = make_fake_service(SERVICE_STATE.ON) service_monitor = self.make_service_monitor([fake_service]) mock_performServiceAction = self.patch(service_monitor, "_performServiceAction") mock_performServiceAction.return_value = succeed(None) active_state = pick_observed_state(but_not={SERVICE_STATE.ON}) service_state = ServiceState(active_state, "dead") mock_getServiceState = self.patch(service_monitor, "getServiceState") mock_getServiceState.return_value = succeed(service_state) with ExpectedException(ServiceActionError): yield service_monitor.restartService(fake_service.name)
def test_reports_services_to_region_when_changed(self): # Pretend we're in a production environment. self.patch(sms, "is_dev_environment").return_value = False protocol, connecting = self.patch_rpc_methods() self.addCleanup((yield connecting)) class ExampleService(AlwaysOnService): name = service_name = snap_service_name = ( factory.make_name("service")) service = ExampleService() # Inveigle this new service into the service monitor. self.addCleanup(service_monitor._services.pop, service.name) service_monitor._services[service.name] = service state = ServiceState(SERVICE_STATE.ON, "running") mock_ensureServices = self.patch(service_monitor, "ensureServices") mock_ensureServices.return_value = succeed({ service.name: state }) client = getRegionClient() rpc_service = Mock() rpc_service.getClientNow.return_value = succeed(client) monitor_service = sms.ServiceMonitorService( rpc_service, Clock()) monitor_service._services = yield monitor_service._buildServices({ service.name: state }) # Force last reported state to dead. That way an update is performed. orig_cached_services = monitor_service._services for ser in orig_cached_services: ser['status'] = 'dead' yield monitor_service.startService() yield monitor_service.stopService() expected_services = list(monitor_service.ALWAYS_RUNNING_SERVICES) expected_services.append({ "name": service.name, "status": "running", "status_info": "", }) self.assertThat( protocol.UpdateServices, MockCalledOnceWith( protocol, system_id=client.localIdent, services=expected_services)) self.assertIsNot(orig_cached_services, monitor_service._services)
def test_buildServices_builds_services_list(self): self.patch(proxyconfig, "is_config_present").return_value = True monitor_service = ServiceMonitorService(Clock()) service = self.pick_service() state = ServiceState(SERVICE_STATE.ON, "running") observed_services = yield monitor_service._buildServices( {service.name: state}) expected_services = [{ "name": service.name, "status": "running", "status_info": "" }] self.assertEqual(expected_services, observed_services)
def test__ensureServices_returns_dict_for_states(self): fake_services = [make_fake_service() for _ in range(3)] expected_states = {} for service in fake_services: active_state = pick_observed_state() process_state = random.choice(["running", "dead"]) expected_states[service.name] = ServiceState( active_state, process_state) service_monitor = self.make_service_monitor(fake_services) self.patch(service_monitor, "ensureService").side_effect = ( lambda name: succeed(expected_states[name])) observed = yield service_monitor.ensureServices() self.assertEquals(expected_states, observed)
def test__reloadService_always_calls_ensureService_then_reloads(self): fake_service = make_fake_service(SERVICE_STATE.ON) service_monitor = self.make_service_monitor([fake_service]) mock_performServiceAction = self.patch(service_monitor, "_performServiceAction") mock_performServiceAction.return_value = succeed(None) mock_ensureService = self.patch(service_monitor, "ensureService") mock_ensureService.return_value = succeed( ServiceState(SERVICE_STATE.ON, "running")) yield service_monitor.reloadService(fake_service.name, if_on=True) self.assertThat(mock_ensureService, MockCalledOnceWith(fake_service.name)) self.assertThat(mock_performServiceAction, MockCalledOnceWith(fake_service, "reload"))
def test_buildServices_adds_services_to_always_running_services(self): monitor_service = sms.ServiceMonitorService(sentinel.client_service, Clock()) service = self.pick_service() state = ServiceState(SERVICE_STATE.ON, "running") observed_services = yield monitor_service._buildServices( {service.name: state}) expected_services = list(monitor_service.ALWAYS_RUNNING_SERVICES) expected_services.append({ "name": service.name, "status": "running", "status_info": "" }) self.assertEquals(expected_services, observed_services)
def test___ensureService_allows_dead_for_off_service(self): service = make_fake_service(SERVICE_STATE.OFF) service_monitor = self.make_service_monitor([service]) mock_getServiceState = self.patch(service_monitor, "getServiceState") mock_getServiceState.return_value = succeed( ServiceState(SERVICE_STATE.DEAD, "Result: exit-code")) with FakeLogger("maas.service_monitor", level=logging.DEBUG) as maaslog: yield service_monitor._ensureService(service) self.assertDocTestMatches( "Service '%s' is %s and '%s'." % (service.service_name, SERVICE_STATE.DEAD, "Result: exit-code"), maaslog.output)
def test__restartService_performs_restart(self): fake_service = make_fake_service(SERVICE_STATE.ON) service_monitor = self.make_service_monitor([fake_service]) mock_performServiceAction = self.patch(service_monitor, "_performServiceAction") mock_performServiceAction.return_value = succeed(None) service_state = ServiceState(SERVICE_STATE.ON, "running") mock_getServiceState = self.patch(service_monitor, "getServiceState") mock_getServiceState.return_value = succeed(service_state) observed = yield service_monitor.restartService(fake_service.name) self.assertEquals(service_state, observed) self.assertThat(mock_getServiceState, MockCalledOnceWith(fake_service.name, now=True)) self.assertThat(mock_performServiceAction, MockCalledOnceWith(fake_service, "restart"))
def test___ensureService_allows_dead_for_off_service(self): service = make_fake_service(SERVICE_STATE.OFF) service_monitor = self.make_service_monitor([service]) mock_getServiceState = self.patch( service_monitor, "getServiceState") mock_getServiceState.return_value = succeed( ServiceState(SERVICE_STATE.DEAD, "Result: exit-code")) log = self.patch(service_monitor_module, 'log') yield service_monitor._ensureService(service) self.assertThat(log.debug, MockCalledOnceWith( "Service '{name}' is {state} and '{process}'.", name=service.service_name, state=SERVICE_STATE.DEAD, process="Result: exit-code"))
def test___ensureService_logs_debug_in_expected_states(self): state = SERVICE_STATE.ON service = make_fake_service(state) service_monitor = self.make_service_monitor([service]) expected_process_state = service_monitor.PROCESS_STATE[state] mock_getServiceState = self.patch( service_monitor, "getServiceState") mock_getServiceState.return_value = succeed( ServiceState(SERVICE_STATE.ON, expected_process_state)) log = self.patch(service_monitor_module, 'log') yield service_monitor._ensureService(service) self.assertThat(log.debug, MockCalledOnceWith( "Service '{name}' is {state} and '{process}'.", name=service.service_name, state=state, process=expected_process_state))
def test___ensureService_logs_debug_in_expected_states(self): state = SERVICE_STATE.ON service = make_fake_service(state) service_monitor = self.make_service_monitor([service]) expected_process_state = service_monitor.PROCESS_STATE[state] mock_getServiceState = self.patch(service_monitor, "getServiceState") mock_getServiceState.return_value = succeed( ServiceState(SERVICE_STATE.ON, expected_process_state)) with FakeLogger("maas.service_monitor", level=logging.DEBUG) as maaslog: yield service_monitor._ensureService(service) self.assertDocTestMatches( "Service '%s' is %s and '%s'." % (service.service_name, state, expected_process_state), maaslog.output)
def test___ensureService_logs_mismatch_for_dead_process_state(self): service = make_fake_service(SERVICE_STATE.OFF) service_monitor = self.make_service_monitor([service]) invalid_process_state = factory.make_name("invalid") mock_getServiceState = self.patch(service_monitor, "getServiceState") mock_getServiceState.return_value = succeed( ServiceState(SERVICE_STATE.DEAD, invalid_process_state)) with FakeLogger("maas.service_monitor", level=logging.WARNING) as maaslog: yield service_monitor._ensureService(service) self.assertDocTestMatches( "Service '%s' is %s but not in the expected state of " "'%s', its current state is '%s'." % (service.service_name, SERVICE_STATE.DEAD.value, service_monitor.PROCESS_STATE[SERVICE_STATE.DEAD], invalid_process_state), maaslog.output)
def test__returns_service_info_message(self): # Make sure any message given by a service gets passed through, except # when the service is dead or off and expected to be on, in which case # a message is manufactured. example_message = factory.make_string(60, spaces=True) example_process_state = factory.make_name("process-state") service = make_fake_service(self.state_expected, example_message) state = ServiceState(self.state_observed, example_process_state) status_string, status_message = yield state.getStatusInfo(service) if self.state_expected == SERVICE_STATE.ON: if self.state_observed == SERVICE_STATE.OFF: self.assertThat(status_message, Equals( service.service_name + " is currently stopped.")) elif self.state_observed == SERVICE_STATE.DEAD: self.assertThat(status_message, Equals( service.service_name + " failed to start, process " "result: (" + example_process_state + ")")) else: self.assertThat(status_message, Equals(example_message)) else: self.assertThat(status_message, Equals(example_message))
def test_doesnt_reports_services_to_region_when_the_same_status(self): # Pretend we're in a production environment. self.patch(sms, "is_dev_environment").return_value = False protocol, connecting = self.patch_rpc_methods() self.addCleanup((yield connecting)) class ExampleService(AlwaysOnService): name = service_name = snap_service_name = ( factory.make_name("service")) service = ExampleService() # Inveigle this new service into the service monitor. self.addCleanup(service_monitor._services.pop, service.name) service_monitor._services[service.name] = service state = ServiceState(SERVICE_STATE.ON, "running") mock_ensureServices = self.patch(service_monitor, "ensureServices") mock_ensureServices.return_value = succeed({ service.name: state }) client = getRegionClient() rpc_service = Mock() rpc_service.getClientNow.return_value = succeed(client) monitor_service = sms.ServiceMonitorService( rpc_service, Clock()) monitor_service._services = yield monitor_service._buildServices({ service.name: state }) yield monitor_service.startService() yield monitor_service.stopService() self.assertThat( protocol.UpdateServices, MockNotCalled())
def test__ensureServices_handles_errors(self): services = make_fake_service(), make_fake_service() service_monitor = self.make_service_monitor(services) # Plant some states into the monitor's memory. service_states = { service.name: ServiceState(pick_observed_state(), random.choice(["running", "dead"])) for service in services } service_monitor._serviceStates.update(service_states) # Make both service monitor checks fail with a distinct error. self.patch(service_monitor, "ensureService") def raise_exception(service_name): raise factory.make_exception(service_name + " broke") def raise_exception_later(service_name): # We use deferLater() to ensure that `raise_exception` is called # asynchronously; this helps to ensure that ensureServices() has # not closed over mutating local state, e.g. a loop variable. return deferLater(reactor, 0, raise_exception, service_name) service_monitor.ensureService.side_effect = raise_exception_later # Capture logs when calling ensureServices(). with FakeLogger("maas.service_monitor") as logger: observed = yield service_monitor.ensureServices() # The errors mean we were returned the states planted earlier. self.assertThat(observed, Equals(service_states)) # The errors were logged with the service name and message. for service in services: self.assertThat( logger.output, Contains( "While monitoring service '%s' an error was encountered: " "%s broke" % (service.name, service.name)))
def test_writes_config_and_restarts_when_omapi_fails(self): write_file = self.patch_sudo_write_file() get_service_state = self.patch_getServiceState() get_service_state.return_value = ServiceState(SERVICE_STATE.ON, "running") restart_service = self.patch_restartService() ensure_service = self.patch_ensureService() update_hosts = self.patch_update_hosts() update_hosts.side_effect = factory.make_exception() failover_peers = make_failover_peer_config() shared_network = make_shared_network() [shared_network] = fix_shared_networks_failover([shared_network], [failover_peers]) old_hosts = [make_host(dhcp_snippets=[]) for _ in range(3)] interface = make_interface() global_dhcp_snippets = make_global_dhcp_snippets() expected_config = factory.make_name("config") self.patch_get_config().return_value = expected_config dhcp_service = dhcp.service_monitor.getServiceByName( self.server.dhcp_service) on = self.patch_autospec(dhcp_service, "on") omapi_key = factory.make_name("omapi_key") old_state = dhcp.DHCPState( omapi_key, [failover_peers], [shared_network], old_hosts, [interface], global_dhcp_snippets, ) dhcp._current_server_state[self.server.dhcp_service] = old_state new_hosts = copy.deepcopy(old_hosts) removed_host = new_hosts.pop() modified_host = new_hosts[0] modified_host["ip"] = factory.make_ip_address( ipv6=self.server.dhcp_service == "DHCPv6") added_host = make_host(dhcp_snippets=[]) new_hosts.append(added_host) with FakeLogger("maas") as logger: yield self.configure( omapi_key, [failover_peers], [shared_network], new_hosts, [interface], global_dhcp_snippets, ) self.assertThat( write_file, MockCallsMatch( call( self.server.config_filename, expected_config.encode("utf-8"), mode=0o640, ), call( self.server.interfaces_filename, interface["name"].encode("utf-8"), mode=0o640, ), ), ) self.assertThat(on, MockCalledOnceWith()) self.assertThat( get_service_state, MockCalledOnceWith(self.server.dhcp_service, now=True), ) self.assertThat(restart_service, MockCalledOnceWith(self.server.dhcp_service)) self.assertThat(ensure_service, MockCalledOnceWith(self.server.dhcp_service)) self.assertThat( update_hosts, MockCalledOnceWith(ANY, [removed_host], [added_host], [modified_host]), ) self.assertEqual( dhcp._current_server_state[self.server.dhcp_service], dhcp.DHCPState( omapi_key, [failover_peers], [shared_network], new_hosts, [interface], global_dhcp_snippets, ), ) self.assertDocTestMatches( "Failed to update all host maps. Restarting DHCPv... " "service to ensure host maps are in-sync.", logger.output, )
def test__writes_config_and_uses_omapi_to_update_hosts(self): write_file = self.patch_sudo_write_file() get_service_state = self.patch_getServiceState() get_service_state.return_value = ServiceState(SERVICE_STATE.ON, "running") restart_service = self.patch_restartService() ensure_service = self.patch_ensureService() update_hosts = self.patch_update_hosts() failover_peers = make_failover_peer_config() shared_network = make_shared_network() [shared_network] = fix_shared_networks_failover([shared_network], [failover_peers]) old_hosts = [make_host(dhcp_snippets=[]) for _ in range(3)] interface = make_interface() global_dhcp_snippets = make_global_dhcp_snippets() expected_config = factory.make_name("config") self.patch_get_config().return_value = expected_config dhcp_service = dhcp.service_monitor.getServiceByName( self.server.dhcp_service) on = self.patch_autospec(dhcp_service, "on") omapi_key = factory.make_name("omapi_key") old_state = dhcp.DHCPState( omapi_key, [failover_peers], [shared_network], old_hosts, [interface], global_dhcp_snippets, ) dhcp._current_server_state[self.server.dhcp_service] = old_state new_hosts = copy.deepcopy(old_hosts) removed_host = new_hosts.pop() modified_host = new_hosts[0] modified_host["ip"] = factory.make_ip_address() added_host = make_host(dhcp_snippets=[]) new_hosts.append(added_host) yield self.configure( omapi_key, [failover_peers], [shared_network], new_hosts, [interface], global_dhcp_snippets, ) self.assertThat( write_file, MockCallsMatch( call( self.server.config_filename, expected_config.encode("utf-8"), mode=0o640, ), call( self.server.interfaces_filename, interface["name"].encode("utf-8"), mode=0o640, ), ), ) self.assertThat(on, MockCalledOnceWith()) self.assertThat( get_service_state, MockCalledOnceWith(self.server.dhcp_service, now=True), ) self.assertThat(restart_service, MockNotCalled()) self.assertThat(ensure_service, MockCalledOnceWith(self.server.dhcp_service)) self.assertThat( update_hosts, MockCalledOnceWith(ANY, [removed_host], [added_host], [modified_host]), ) self.assertEquals( dhcp._current_server_state[self.server.dhcp_service], dhcp.DHCPState( omapi_key, [failover_peers], [shared_network], new_hosts, [interface], global_dhcp_snippets, ), )