def test_service_manager_start_stop(self): """ This test exercises the ServiceManager functions, but doesn't really verify functionality beyond the tests above """ mgr = ServiceManager( ['dummy1', 'dummy2'], init_system='systemd', service_poller=MagicMock(), ) self._loop.run_until_complete(mgr.start_services()) self.subprocess_mock.return_value = b'active\n' self.assertEqual( mgr._service_control['dummy1'].status(), ServiceState.Active, ) self.assertEqual( mgr._service_control['dummy2'].status(), ServiceState.Active, ) self._loop.run_until_complete(mgr.stop_services()) self.subprocess_mock.return_value = b'inactive\n' self.assertEqual( mgr._service_control['dummy1'].status(), ServiceState.Inactive, ) self.assertEqual( mgr._service_control['dummy2'].status(), ServiceState.Inactive, )
def test_service_manager_start_stop(self): """ This test exercises the ServiceManager functions, but doesn't really verify functionality beyond the tests above """ mgr = ServiceManager(['dummy1', 'dummy2'], init_system='docker', service_poller=MagicMock()) try: self._loop.run_until_complete(mgr.start_services()) except FileNotFoundError: pass self.subprocess_mock.return_value = b'[{"State": {"Status": "running"}}]' self.assertEqual(mgr._service_control['dummy1'].status(), ServiceState.Active) self.assertEqual(mgr._service_control['dummy2'].status(), ServiceState.Active) try: self._loop.run_until_complete(mgr.stop_services()) except FileNotFoundError: pass self.subprocess_mock.return_value = b'[{"State": {"Status": "exited"}}]' self.assertEqual(mgr._service_control['dummy1'].status(), ServiceState.Inactive) self.assertEqual(mgr._service_control['dummy2'].status(), ServiceState.Inactive)
def test_dynamic_service_manager_start_stop(self): """ This test exercises the ServiceManager functions, but doesn't really verify functionality beyond the tests above """ # Skip for integration tests, otherwise will fail on systems without # runit if self.is_integration_test: return mgr = ServiceManager( ['dummy1', 'dummy2'], 'runit', MagicMock(), ['redirectd'], [] ) self.subprocess_mock.return_value = ( 'down: e2e_controller: 268195s; run: log: (pid 2275) 268195s\n' ) self.assertEqual(mgr._service_control['redirectd'].status(), ServiceState.Inactive) mgr.update_dynamic_services(['redirectd']) self.subprocess_mock.return_value = ( 'run: e2e_controller: 268195s; run: log: (pid 2275) 268195s\n' ) self.assertEqual(mgr._service_control['redirectd'].status(), ServiceState.Active) mgr.update_dynamic_services([]) self.subprocess_mock.return_value = ( 'down: e2e_controller: 268195s; run: log: (pid 2275) 268195s\n' ) self.assertEqual(mgr._service_control['redirectd'].status(), ServiceState.Inactive)
def test_dynamic_service_manager_start_stop(self): """ This test exercises the ServiceManager functions, but doesn't really verify functionality beyond the tests above """ mgr = ServiceManager(['dummy1', 'dummy2'], 'systemd', MagicMock(), ['redirectd'], []) self.subprocess_mock.return_value = b'inactive\n' self.assertEqual(mgr._service_control['redirectd'].status(), ServiceState.Inactive) self._loop.run_until_complete( mgr.update_dynamic_services(['redirectd'])) self.subprocess_mock.return_value = b'active\n' self.assertEqual(mgr._service_control['redirectd'].status(), ServiceState.Active) self._loop.run_until_complete(mgr.update_dynamic_services([])) self.subprocess_mock.return_value = b'inactive\n' self.assertEqual(mgr._service_control['redirectd'].status(), ServiceState.Inactive)
def setUp(self): """ Run before each test """ # Only patch if this is a unit test. Otherwise create dummy mock so # code that affects the mock can run. self.subprocess_mock = MagicMock(return_value='') if not os.environ.get('MAGMA_INTEGRATION_TEST'): subprocess.check_output = self.subprocess_mock # Ensure test process is stopped self.dummy_service = ServiceManager.ServiceControl( name='dummy_service', init_system_spec=ServiceManager.SystemdInitSystem) self.dummy_service.stop_process()
def setUp(self): """ Run before each test """ self.is_integration_test = bool( os.environ.get('MAGMA_INTEGRATION_TEST'), ) # Only patch if this is a unit test. Otherwise create dummy mock so # code that affects the mock can run. self.subprocess_mock = MagicMock(return_value='') if not self.is_integration_test: subprocess.check_output = self.subprocess_mock if not self.is_integration_test: self.dummy_service = ServiceManager.ServiceControl( name='dummy_service', init_system_spec=ServiceManager.RunitInitSystem, ) self.dummy_service.stop_process()
def setUp(self): """ Run before each test """ self._loop = asyncio.new_event_loop() asyncio.set_event_loop(self._loop) # Only patch if this is a unit test. Otherwise create dummy mock so # code that affects the mock can run. self.subprocess_mock = MagicMock(return_value='') if not os.environ.get('MAGMA_INTEGRATION_TEST'): subprocess.check_output = self.subprocess_mock # Ensure test process is stopped self.dummy_service = ServiceManager.ServiceControl( name='dummy_service', init_system_spec=ServiceManager.DockerInitSystem) try: self._loop.run_until_complete(self.dummy_service.stop_process()) except FileNotFoundError: pass
def main(): """ Main magmad function """ service = MagmaService('magmad', mconfigs_pb2.MagmaD()) # Optionally pipe errors to Sentry sentry_init(service_name=service.name) logging.info('Starting magmad for UUID: %s', snowflake.make_snowflake()) # Create service manager services = service.config.get('magma_services') init_system = service.config.get('init_system', 'systemd') registered_dynamic_services = service.config.get( 'registered_dynamic_services', [], ) enabled_dynamic_services = [] if service.mconfig is not None: enabled_dynamic_services = service.mconfig.dynamic_services # Poll the services' Service303 interface service_poller = ServicePoller( service.loop, service.config, enabled_dynamic_services, ) service_poller.start() service_manager = ServiceManager( services, init_system, service_poller, registered_dynamic_services, enabled_dynamic_services, ) # Get metrics service config metrics_config = service.config.get('metricsd') metrics_services = metrics_config['services'] collect_interval = metrics_config['collect_interval'] sync_interval = metrics_config['sync_interval'] grpc_timeout = metrics_config['grpc_timeout'] grpc_msg_size = metrics_config.get('max_grpc_msg_size_mb', 4) metrics_post_processor_fn = metrics_config.get('post_processing_fn') metric_scrape_targets = [ ScrapeTarget(t['url'], t['name'], t['interval']) for t in metrics_config.get('metric_scrape_targets', []) ] # Create local metrics collector metrics_collector = MetricsCollector( services=metrics_services, collect_interval=collect_interval, sync_interval=sync_interval, grpc_timeout=grpc_timeout, grpc_max_msg_size_mb=grpc_msg_size, loop=service.loop, post_processing_fn=get_metrics_postprocessor_fn( metrics_post_processor_fn, ), scrape_targets=metric_scrape_targets, ) # Poll and sync the metrics collector loops metrics_collector.run() # Start a background thread to stream updates from the cloud stream_client = None if service.config.get('enable_config_streamer', False): stream_client = StreamerClient( { CONFIG_STREAM_NAME: ConfigManager( services, service_manager, service, MconfigManagerImpl(), ), }, service.loop, ) # Create sync rpc client with a heartbeat of 30 seconds (timeout = 60s) sync_rpc_client = None if service.config.get('enable_sync_rpc', False): sync_rpc_client = SyncRPCClient( service.loop, 30, service.config.get('print_grpc_payload', False), ) first_time_bootstrap = True # This is called when bootstrap succeeds and when _bootstrap_check is # invoked but bootstrap is not needed. If it's invoked right after certs # are generated, certs_generated is true, control_proxy will restart. async def bootstrap_success_cb(certs_generated: bool): nonlocal first_time_bootstrap if first_time_bootstrap: if stream_client: stream_client.start() if sync_rpc_client: sync_rpc_client.start() first_time_bootstrap = False if certs_generated: svcs_to_restart = [] if 'control_proxy' in services: svcs_to_restart.append('control_proxy') # fluent-bit caches TLS client certs in memory, so we need to # restart it whenever the certs change fresh_mconfig = get_mconfig_manager().load_service_mconfig( 'magmad', mconfigs_pb2.MagmaD(), ) dynamic_svcs = fresh_mconfig.dynamic_services or [] if 'td-agent-bit' in dynamic_svcs: svcs_to_restart.append('td-agent-bit') await service_manager.restart_services(services=svcs_to_restart) # Create bootstrap manager bootstrap_manager = BootstrapManager(service, bootstrap_success_cb) # Initialize kernel version poller if it is enabled kernel_version_poller = None if service.config.get('enable_kernel_version_checking', False): kernel_version_poller = KernelVersionsPoller(service) kernel_version_poller.start() # gateway status generator to bundle various information about this # gateway into an object. gateway_status_factory = GatewayStatusFactory( service=service, service_poller=service_poller, kernel_version_poller=kernel_version_poller, ) # _grpc_client_manager to manage grpc client recycling grpc_client_manager = GRPCClientManager( service_name="state", service_stub=StateServiceStub, max_client_reuse=60, ) # Initialize StateReporter state_reporter = StateReporter( config=service.config, mconfig=service.mconfig, loop=service.loop, bootstrap_manager=bootstrap_manager, gw_status_factory=gateway_status_factory, grpc_client_manager=grpc_client_manager, ) # Initialize ServiceHealthWatchdog service_health_watchdog = ServiceHealthWatchdog( config=service.config, loop=service.loop, service_poller=service_poller, service_manager=service_manager, ) # Start _bootstrap_manager bootstrap_manager.start_bootstrap_manager() # Start all services when magmad comes up service.loop.create_task(service_manager.start_services()) # Start state reporting loop state_reporter.start() # Start service timeout health check loop service_health_watchdog.start() # Start upgrade manager loop if service.config.get('enable_upgrade_manager', False): upgrader = _get_upgrader_impl(service) service.loop.create_task(start_upgrade_loop(service, upgrader)) # Start network health metric collection if service.config.get('enable_network_monitor', False): service.loop.create_task(metrics_collection_loop(service.config)) # Create generic command executor command_executor = None if service.config.get('generic_command_config', None): command_executor = get_command_executor_impl(service) # Start loop to monitor unattended upgrade status service.loop.create_task(monitor_unattended_upgrade_status()) # Add all servicers to the server magmad_servicer = MagmadRpcServicer( service, services, service_manager, get_mconfig_manager(), command_executor, service.loop, service.config.get('print_grpc_payload', False), ) magmad_servicer.add_to_server(service.rpc_server) if SDWatchdog.has_notify(): # Create systemd watchdog sdwatchdog = SDWatchdog( tasks=[bootstrap_manager, state_reporter], update_status=True, ) # Start watchdog loop service.loop.create_task(sdwatchdog.run()) # Run the service loop service.run() # Cleanup the service service.close()