def setUp(self):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
self._loop = loop
self._sync_rpc_client = SyncRPCClient(loop=loop, response_timeout=3)
self._sync_rpc_client._conn_closed_table = {12345: False}
ServiceRegistry.add_service('test', '0.0.0.0', 0)
ServiceRegistry._PROXY_CONFIG = {
'local_port': 2345,
'cloud_address': 'test',
'proxy_cloud_connections': True
}
self._req_body = GatewayRequest(gwId="test id",
authority='mobility',
path='/magma.MobilityService'
'/ListAddedIPv4Blocks',
headers={
'te': 'trailers',
'content-type': 'application/grpc',
'user-agent': 'grpc-python/1.4.0',
'grpc-accept-encoding': 'identity'
},
payload=bytes.fromhex('0000000000'))
self._expected_resp = GatewayResponse(status="400",
headers={"test_key": "test_val"},
payload=b'\x00'
b'\x00\x00\x00\n\n\x08')
self._expected_err_msg = "test error"
class SyncRPCClientTests(unittest.TestCase):
"""
Tests for the SyncRPCClient
"""
def setUp(self):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
self._loop = loop
self._sync_rpc_client = SyncRPCClient(loop=loop, response_timeout=3)
self._sync_rpc_client._conn_closed_table = {12345: False}
ServiceRegistry.add_service('test', '0.0.0.0', 0)
ServiceRegistry._PROXY_CONFIG = {
'local_port': 2345,
'cloud_address': 'test',
'proxy_cloud_connections': True
}
self._req_body = GatewayRequest(gwId="test id",
authority='mobility',
path='/magma.MobilityService'
'/ListAddedIPv4Blocks',
headers={
'te': 'trailers',
'content-type': 'application/grpc',
'user-agent': 'grpc-python/1.4.0',
'grpc-accept-encoding': 'identity'
},
payload=bytes.fromhex('0000000000'))
self._expected_resp = GatewayResponse(status="400",
headers={"test_key": "test_val"},
payload=b'\x00'
b'\x00\x00\x00\n\n\x08')
self._expected_err_msg = "test error"
def test_forward_request_conn_closed(self):
self._sync_rpc_client.forward_request(
SyncRPCRequest(reqId=12345, connClosed=True))
self.assertEqual(self._sync_rpc_client._conn_closed_table[12345], True)
def test_send_sync_rpc_response(self):
expected = SyncRPCResponse(reqId=123, respBody=self._expected_resp)
self._sync_rpc_client._response_queue.put(expected)
res = self._sync_rpc_client.send_sync_rpc_response()
actual = next(res)
self.assertEqual(expected, actual)
expected = SyncRPCResponse(heartBeat=True)
actual = next(res)
self.assertEqual(expected, actual)
def test_retry_connect_sleep(self):
self._sync_rpc_client._current_delay = 0
for i in range(5):
self._sync_rpc_client._retry_connect_sleep()
if i == 4:
self.assertEqual(self._sync_rpc_client.RETRY_MAX_DELAY_SECS,
self._sync_rpc_client._current_delay)
else:
self.assertEqual(2**i, self._sync_rpc_client._current_delay)
def main():
"""
Main magmad function
"""
service = MagmaService('magmad', mconfigs_pb2.MagmaD())
# Optionally pipe errors to Sentry
sentry_init(service_name=service.name)
logging.info('Starting magmad for UUID: %s', snowflake.make_snowflake())
# Create service manager
services = service.config.get('magma_services')
init_system = service.config.get('init_system', 'systemd')
registered_dynamic_services = service.config.get(
'registered_dynamic_services',
[],
)
enabled_dynamic_services = []
if service.mconfig is not None:
enabled_dynamic_services = service.mconfig.dynamic_services
# Poll the services' Service303 interface
service_poller = ServicePoller(
service.loop,
service.config,
enabled_dynamic_services,
)
service_poller.start()
service_manager = ServiceManager(
services,
init_system,
service_poller,
registered_dynamic_services,
enabled_dynamic_services,
)
# Get metrics service config
metrics_config = service.config.get('metricsd')
metrics_services = metrics_config['services']
collect_interval = metrics_config['collect_interval']
sync_interval = metrics_config['sync_interval']
grpc_timeout = metrics_config['grpc_timeout']
grpc_msg_size = metrics_config.get('max_grpc_msg_size_mb', 4)
metrics_post_processor_fn = metrics_config.get('post_processing_fn')
metric_scrape_targets = [
ScrapeTarget(t['url'], t['name'], t['interval'])
for t in metrics_config.get('metric_scrape_targets', [])
]
# Create local metrics collector
metrics_collector = MetricsCollector(
services=metrics_services,
collect_interval=collect_interval,
sync_interval=sync_interval,
grpc_timeout=grpc_timeout,
grpc_max_msg_size_mb=grpc_msg_size,
loop=service.loop,
post_processing_fn=get_metrics_postprocessor_fn(
metrics_post_processor_fn, ),
scrape_targets=metric_scrape_targets,
)
# Poll and sync the metrics collector loops
metrics_collector.run()
# Start a background thread to stream updates from the cloud
stream_client = None
if service.config.get('enable_config_streamer', False):
stream_client = StreamerClient(
{
CONFIG_STREAM_NAME:
ConfigManager(
services,
service_manager,
service,
MconfigManagerImpl(),
),
},
service.loop,
)
# Create sync rpc client with a heartbeat of 30 seconds (timeout = 60s)
sync_rpc_client = None
if service.config.get('enable_sync_rpc', False):
sync_rpc_client = SyncRPCClient(
service.loop,
30,
service.config.get('print_grpc_payload', False),
)
first_time_bootstrap = True
# This is called when bootstrap succeeds and when _bootstrap_check is
# invoked but bootstrap is not needed. If it's invoked right after certs
# are generated, certs_generated is true, control_proxy will restart.
async def bootstrap_success_cb(certs_generated: bool):
nonlocal first_time_bootstrap
if first_time_bootstrap:
if stream_client:
stream_client.start()
if sync_rpc_client:
sync_rpc_client.start()
first_time_bootstrap = False
if certs_generated:
svcs_to_restart = []
if 'control_proxy' in services:
svcs_to_restart.append('control_proxy')
# fluent-bit caches TLS client certs in memory, so we need to
# restart it whenever the certs change
fresh_mconfig = get_mconfig_manager().load_service_mconfig(
'magmad',
mconfigs_pb2.MagmaD(),
)
dynamic_svcs = fresh_mconfig.dynamic_services or []
if 'td-agent-bit' in dynamic_svcs:
svcs_to_restart.append('td-agent-bit')
await service_manager.restart_services(services=svcs_to_restart)
# Create bootstrap manager
bootstrap_manager = BootstrapManager(service, bootstrap_success_cb)
# Initialize kernel version poller if it is enabled
kernel_version_poller = None
if service.config.get('enable_kernel_version_checking', False):
kernel_version_poller = KernelVersionsPoller(service)
kernel_version_poller.start()
# gateway status generator to bundle various information about this
# gateway into an object.
gateway_status_factory = GatewayStatusFactory(
service=service,
service_poller=service_poller,
kernel_version_poller=kernel_version_poller,
)
# _grpc_client_manager to manage grpc client recycling
grpc_client_manager = GRPCClientManager(
service_name="state",
service_stub=StateServiceStub,
max_client_reuse=60,
)
# Initialize StateReporter
state_reporter = StateReporter(
config=service.config,
mconfig=service.mconfig,
loop=service.loop,
bootstrap_manager=bootstrap_manager,
gw_status_factory=gateway_status_factory,
grpc_client_manager=grpc_client_manager,
)
# Initialize ServiceHealthWatchdog
service_health_watchdog = ServiceHealthWatchdog(
config=service.config,
loop=service.loop,
service_poller=service_poller,
service_manager=service_manager,
)
# Start _bootstrap_manager
bootstrap_manager.start_bootstrap_manager()
# Start all services when magmad comes up
service.loop.create_task(service_manager.start_services())
# Start state reporting loop
state_reporter.start()
# Start service timeout health check loop
service_health_watchdog.start()
# Start upgrade manager loop
if service.config.get('enable_upgrade_manager', False):
upgrader = _get_upgrader_impl(service)
service.loop.create_task(start_upgrade_loop(service, upgrader))
# Start network health metric collection
if service.config.get('enable_network_monitor', False):
service.loop.create_task(metrics_collection_loop(service.config))
# Create generic command executor
command_executor = None
if service.config.get('generic_command_config', None):
command_executor = get_command_executor_impl(service)
# Start loop to monitor unattended upgrade status
service.loop.create_task(monitor_unattended_upgrade_status())
# Add all servicers to the server
magmad_servicer = MagmadRpcServicer(
service,
services,
service_manager,
get_mconfig_manager(),
command_executor,
service.loop,
service.config.get('print_grpc_payload', False),
)
magmad_servicer.add_to_server(service.rpc_server)
if SDWatchdog.has_notify():
# Create systemd watchdog
sdwatchdog = SDWatchdog(
tasks=[bootstrap_manager, state_reporter],
update_status=True,
)
# Start watchdog loop
service.loop.create_task(sdwatchdog.run())
# Run the service loop
service.run()
# Cleanup the service
service.close()
class SyncRPCClientTests(unittest.TestCase):
"""
Tests for the SyncRPCClient
"""
def setUp(self):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
self._loop = loop
self._sync_rpc_client = SyncRPCClient(loop=loop, response_timeout=3)
ServiceRegistry.add_service('test', '0.0.0.0', 0)
ServiceRegistry._PROXY_CONFIG = {
'local_port': 2345,
'cloud_address': 'test',
'proxy_cloud_connections': True
}
self._req_body = GatewayRequest(gwId="test id",
authority='mobility',
path='/magma.MobilityService'
'/ListAddedIPv4Blocks',
headers={
'te': 'trailers',
'content-type': 'application/grpc',
'user-agent': 'grpc-python/1.4.0',
'grpc-accept-encoding': 'identity'
},
payload=bytes.fromhex('0000000000'))
self._expected_resp = GatewayResponse(status="400",
headers={"test_key": "test_val"},
payload=b'\x00'
b'\x00\x00\x00\n\n\x08')
self._expected_err_msg = "test error"
def test_send_request_done(self):
"""
Test if send_request_done puts the right SyncRPCResponse in
response_queue based on the result of future.
Returns: None
"""
self.assertTrue(self._sync_rpc_client._response_queue.empty())
req_id = 356
future = MockFuture(is_error=False,
expected_result=self._expected_resp,
expected_err_msg="")
# future has result, send_request_done should enqueue a SyncRPCResponse
# with the req_id, and the expected_resp set in MockFuture
self._sync_rpc_client.send_request_done(req_id, future)
self.assertFalse(self._sync_rpc_client._response_queue.empty())
res = self._sync_rpc_client._response_queue.get(block=False)
self.assertEqual(
res,
SyncRPCResponse(reqId=req_id,
respBody=self._expected_resp,
heartBeat=False))
self.assertTrue(self._sync_rpc_client._response_queue.empty())
req_id = 234
expected_err_resp = GatewayResponse(err=self._expected_err_msg)
future = MockFuture(is_error=True,
expected_result=GatewayResponse(),
expected_err_msg=self._expected_err_msg)
self._sync_rpc_client.send_request_done(req_id, future)
res = self._sync_rpc_client._response_queue.get(block=False)
self.assertEqual(
res,
SyncRPCResponse(reqId=req_id,
respBody=expected_err_resp,
heartBeat=False))
def test_send_sync_rpc_response(self):
expected = SyncRPCResponse(reqId=123, respBody=self._expected_resp)
self._sync_rpc_client._response_queue.put(expected)
res = self._sync_rpc_client.send_sync_rpc_response()
actual = next(res)
self.assertEqual(expected, actual)
expected = SyncRPCResponse(heartBeat=True)
actual = next(res)
self.assertEqual(expected, actual)
def test_retry_connect_sleep(self):
self._sync_rpc_client._current_delay = 0
for i in range(5):
self._sync_rpc_client._set_connect_time()
self._sync_rpc_client._retry_connect_sleep()
if i == 4:
self.assertEqual(self._sync_rpc_client.RETRY_MAX_DELAY_SECS,
self._sync_rpc_client._current_delay)
else:
self.assertEqual(2**i, self._sync_rpc_client._current_delay)