def deserialize_mconfig(
self,
serialized_value: str,
allow_unknown_fields: bool = True,
) -> GatewayConfigs:
# First parse as JSON in case there are types unrecognized by
# protobuf symbol database
json_mconfig = json.loads(serialized_value)
cfgs_by_key_json = json_mconfig.get('configs_by_key', {})
cfgs_by_key_json.update(json_mconfig.get('configsByKey', {}))
filtered_cfgs_by_key = filter_configs_by_key(cfgs_by_key_json)
# Set configs to filtered map, re-dump and parse
if 'configs_by_key' in json_mconfig:
json_mconfig.pop('configs_by_key')
json_mconfig['configsByKey'] = filtered_cfgs_by_key
json_mconfig_dumped = json.dumps(json_mconfig)
# Workaround for outdated protobuf library on sandcastle
if allow_unknown_fields:
return json_format.Parse(
json_mconfig_dumped,
GatewayConfigs(),
ignore_unknown_fields=True,
)
else:
return json_format.Parse(json_mconfig_dumped, GatewayConfigs())
def _get_old_and_new_mconfig_fixtures():
"""
Returns (old_mconfig, new_mconfig) where new_mconfig updates metricsd
config and magmad config. new_mconfig also adds a config for an
unrecognized service keyed by 'mock'
"""
old_magmad_any = Any()
old_magmad_config = MagmaD(checkin_interval=123)
old_magmad_any.Pack(old_magmad_config)
new_magmad_any = Any()
new_magmad_config = MagmaD(
checkin_interval=42,
feature_flags={'kafka_config_streamer': True},
)
new_magmad_any.Pack(new_magmad_config)
old_metricsd_any = Any()
old_metricsd_config = MetricsD(log_level=common_pb2.ERROR)
old_metricsd_any.Pack(old_metricsd_config)
new_metricsd_any = Any()
new_metricsd_config = MetricsD(log_level=common_pb2.INFO)
new_metricsd_any.Pack(new_metricsd_config)
# Unrecognized service, good type
mock_service_any = Any()
mock_service_config = MetricsD(log_level=common_pb2.FATAL)
mock_service_any.Pack(mock_service_config)
old_mconfig = OffsetGatewayConfigs(
offset=42,
configs=GatewayConfigs(configs_by_key={
'magmad': old_magmad_any,
'metricsd': old_metricsd_any,
}, ),
)
new_mconfig = OffsetGatewayConfigs(
offset=43,
configs=GatewayConfigs(configs_by_key={
'magmad': new_magmad_any,
'metricsd': new_metricsd_any,
'mock': mock_service_any,
}, ),
)
return old_mconfig, new_mconfig
def test_update(self, config_mock):
"""
Test that mconfig updates are handled correctly
"""
# Set up fixture data
# Update will simulate gateway moving from
# test_mconfig -> updated_mconfig
TestUpdate = namedtuple('TestUpdate', ['value', 'key'])
test_mconfig = GatewayConfigs()
updated_mconfig = GatewayConfigs()
some_any = Any()
magmad = MagmaD(log_level=1)
some_any.Pack(magmad)
test_mconfig.configs_by_key['magmad'].CopyFrom(some_any)
updated_mconfig.configs_by_key['magmad'].CopyFrom(some_any)
metricsd = MetricsD(log_level=2)
some_any.Pack(metricsd)
test_mconfig.configs_by_key['metricsd'].CopyFrom(some_any)
metricsd = MetricsD(log_level=3)
some_any.Pack(metricsd)
updated_mconfig.configs_by_key['metricsd'].CopyFrom(some_any)
# Set up mock dependencies
config_mock.return_value = {
'magma_services': ['magmad', 'metricsd'],
}
@asyncio.coroutine
def _mock_restart_services(): return "blah"
service_manager_mock = MagicMock()
magmad_service_mock = MagicMock()
mconfig_manager_mock = MconfigManagerImpl()
load_mock = patch.object(
mconfig_manager_mock,
'load_mconfig', MagicMock(return_value=test_mconfig),
)
update_mock = patch.object(
mconfig_manager_mock,
'update_stored_mconfig', Mock(),
)
restart_service_mock = patch.object(
service_manager_mock,
'restart_services', MagicMock(wraps=_mock_restart_services),
)
with load_mock as loader, update_mock as updater, \
restart_service_mock as restarter:
loop = asyncio.new_event_loop()
config_manager = ConfigManager(
['magmad', 'metricsd'], service_manager_mock,
magmad_service_mock, mconfig_manager_mock,
allow_unknown_fields=False,
loop=loop,
)
# Verify that config update restarts all services
update_str = MessageToJson(updated_mconfig)
updates = [
TestUpdate(value='', key='some key'),
TestUpdate(
value=update_str.encode('utf-8'),
key='last key',
),
]
config_manager.process_update(CONFIG_STREAM_NAME, updates, False)
# Only metricsd config was updated, hence should be restarted
loader.assert_called_once_with()
restarter.assert_called_once_with(['metricsd'])
updater.assert_called_once_with(update_str)
def test_update(self, config_mock):
"""
Test that mconfig updates are handled correctly
"""
# Set up fixture data
# Update will simulate gateway moving from
# test_mconfig -> updated_mconfig
test_mconfig = GatewayConfigs()
updated_mconfig = GatewayConfigs()
some_any = Any()
magmad = MagmaD(log_level=1)
some_any.Pack(magmad)
test_mconfig.configs_by_key['magmad'].CopyFrom(some_any)
updated_mconfig.configs_by_key['magmad'].CopyFrom(some_any)
metricsd = MetricsD(log_level=2)
some_any.Pack(metricsd)
test_mconfig.configs_by_key['metricsd'].CopyFrom(some_any)
metricsd = MetricsD(log_level=3)
some_any.Pack(metricsd)
updated_mconfig.configs_by_key['metricsd'].CopyFrom(some_any)
# Set up mock dependencies
config_mock.return_value = {
'magma_services': ['magmad', 'metricsd'],
}
@asyncio.coroutine
def _mock_restart_services():
return "blah"
@asyncio.coroutine
def _mock_update_dynamic_services():
return "mockResponse"
service_manager_mock = MagicMock()
magmad_service_mock = MagicMock()
mconfig_manager_mock = MconfigManagerImpl()
load_mock = patch.object(
mconfig_manager_mock,
'load_mconfig', MagicMock(return_value=test_mconfig),
)
update_mock = patch.object(
mconfig_manager_mock,
'update_stored_mconfig', Mock(),
)
restart_service_mock = patch.object(
service_manager_mock,
'restart_services', MagicMock(wraps=_mock_restart_services),
)
update_dynamic_services_mock = patch.object(
service_manager_mock,
'update_dynamic_services', MagicMock(wraps=_mock_update_dynamic_services),
)
processed_updates_mock = patch('magma.magmad.events.processed_updates')
class ServiceMconfigMock:
def __init__(self, service, mconfig_struct):
pass
dynamic_services = []
mock_mcfg = patch('magma.magmad.config_manager.load_service_mconfig', MagicMock(wraps=ServiceMconfigMock))
with load_mock as loader,\
update_mock as updater, \
restart_service_mock as restarter,\
update_dynamic_services_mock as dynamic_services,\
mock_mcfg as mock_c,\
processed_updates_mock as processed_updates:
loop = asyncio.new_event_loop()
config_manager = ConfigManager(
['magmad', 'metricsd'], service_manager_mock,
magmad_service_mock, mconfig_manager_mock,
allow_unknown_fields=False,
loop=loop,
)
# Process an empty set of updates
updates = []
config_manager.process_update(CONFIG_STREAM_NAME, updates, False)
# No services should be updated or restarted due to empty updates
restarter.assert_not_called()
updater.assert_not_called()
# Verify that config update restarts all services
update_str = MessageToJson(updated_mconfig)
updates = [
DataUpdate(value=''.encode('utf-8'), key='some key'),
DataUpdate(
value=update_str.encode('utf-8'),
key='last key',
),
]
config_manager.process_update(CONFIG_STREAM_NAME, updates, False)
# Only metricsd config was updated, hence should be restarted
self.assertEqual(loader.call_count, 1)
restarter.assert_called_once_with(['metricsd'])
updater.assert_called_once_with(update_str)
configs_by_service = {
'magmad': updated_mconfig.configs_by_key['magmad'],
'metricsd': updated_mconfig.configs_by_key['metricsd'],
}
processed_updates.assert_called_once_with(configs_by_service)
restarter.reset_mock()
updater.reset_mock()
processed_updates.reset_mock()
updated_mconfig.configs_by_key['shared_mconfig'].CopyFrom(some_any)
update_str = MessageToJson(updated_mconfig)
updates = [
DataUpdate(
value=update_str.encode('utf-8'),
key='last key',
),
]
config_manager.process_update(CONFIG_STREAM_NAME, updates, False)
# shared config update should restart all services
restarter.assert_called_once_with(['magmad', 'metricsd'])
updater.assert_called_once_with(update_str)
dynamic_services.assert_called_once_with([])
processed_updates.assert_called_once_with(configs_by_service)
self.assertEqual(mock_c.call_count, 1)
restarter.reset_mock()
updater.reset_mock()
processed_updates.reset_mock()
dynamic_services.reset_mock()