def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_proto_serializer(),
get_proto_deserializer(PolicyRule),
)
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_proto_serializer(),
get_proto_deserializer(ChargingRuleNameSet),
)
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_proto_serializer(),
get_proto_deserializer(RatingGroup),
)
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_proto_serializer(),
get_proto_deserializer(SubscriberPolicySet),
)
self._clear()
def _parse_state_proto(self, key_type, value):
proto = self.STATE_PROTOS.get(key_type.lower())
if proto:
deserializer = get_proto_deserializer(proto)
version_deserializer = get_proto_version_deserializer()
print(deserializer(value))
print('==================')
print('State version: %s' % version_deserializer(value))
else:
raise AttributeError('Key not found on redis')
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_proto_serializer(),
get_proto_deserializer(InstalledPolicies),
)
# TODO: Remove when sessiond becomes stateless
self._clear()
def get_proto_redis_dicts(config):
redis_dicts = []
state_protos = config.get('state_protos', []) or []
for proto_cfg in state_protos:
is_invalid_cfg = 'proto_msg' not in proto_cfg or \
'proto_file' not in proto_cfg or \
'redis_key' not in proto_cfg or \
'state_scope' not in proto_cfg
if is_invalid_cfg:
logging.warning(
"Invalid proto config found in state_protos "
"configuration: %s",
proto_cfg,
)
continue
try:
proto_module = importlib.import_module(proto_cfg['proto_file'])
msg = getattr(proto_module, proto_cfg['proto_msg'])
redis_key = proto_cfg['redis_key']
logging.info(
'Initializing RedisSerde for proto state %s',
proto_cfg['redis_key'],
)
serde = RedisSerde(
redis_key,
get_proto_serializer(),
get_proto_deserializer(msg),
)
redis_dict = StateDict(
serde,
proto_cfg['state_scope'],
PROTO_FORMAT,
)
redis_dicts.append(redis_dict)
except (ImportError, AttributeError) as err:
logging.error(err)
return redis_dicts
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
service = MagicMock()
service.config = {
# Replicate arbitrary orc8r protos
'state_protos': [{
'proto_file': 'orc8r.protos.common_pb2',
'proto_msg': 'NetworkID',
'redis_key': NID_TYPE,
'state_scope': 'network'
}, {
'proto_file': 'orc8r.protos.common_pb2',
'proto_msg': 'IDList',
'redis_key': IDList_TYPE,
'state_scope': 'gateway'
}, {
'proto_file': 'orc8r.protos.service303_pb2',
'proto_msg': 'LogVerbosity',
'redis_key': LOG_TYPE,
'state_scope': 'gateway'
}],
'json_state': [{
'redis_key': FOO_TYPE,
'state_scope': 'network'
}]
}
service.loop = self.loop
# Bind the rpc server to a free port
self._rpc_server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10))
port = self._rpc_server.add_insecure_port('0.0.0.0:0')
# Add the servicer
self._servicer = DummyStateServer()
self._servicer.add_to_server(self._rpc_server)
self._rpc_server.start()
# Create a rpc stub
self.channel = grpc.insecure_channel('0.0.0.0:{}'.format(port))
serde1 = RedisSerde(NID_TYPE, get_proto_serializer(),
get_proto_deserializer(NetworkID))
serde2 = RedisSerde(IDList_TYPE, get_proto_serializer(),
get_proto_deserializer(IDList))
serde3 = RedisSerde(LOG_TYPE, get_proto_serializer(),
get_proto_deserializer(LogVerbosity))
serde4 = RedisSerde(FOO_TYPE, get_json_serializer(),
get_json_deserializer())
self.nid_client = RedisFlatDict(get_default_client(), serde1)
self.idlist_client = RedisFlatDict(get_default_client(), serde2)
self.log_client = RedisFlatDict(get_default_client(), serde3)
self.foo_client = RedisFlatDict(get_default_client(), serde4)
# Set up and start state replicating loop
grpc_client_manager = GRPCClientManager(
service_name="state",
service_stub=StateServiceStub,
max_client_reuse=60,
)
garbage_collector = GarbageCollector(service, grpc_client_manager)
self.state_replicator = StateReplicator(
service=service,
garbage_collector=garbage_collector,
grpc_client_manager=grpc_client_manager,
)
self.state_replicator.start()
class StateCLI(object):
"""
CLI for debugging current Magma services state and displaying it
in readable manner.
"""
STATE_DESERIALIZERS = {
'assigned_ip_blocks': deserialize_ip_block,
'ip_states': deserialize_ip_desc,
'sessions': _deserialize_session_json,
'rule_names': get_json_deserializer(),
'rule_ids': get_json_deserializer(),
'rule_versions': get_json_deserializer(),
'rules': get_proto_deserializer(PolicyRule),
}
STATE_PROTOS = {
'mme_nas_state': MmeNasState,
'spgw_state': SpgwState,
's1ap_state': S1apState,
's1ap_imsi_map': S1apImsiMap,
'mme': UeContext,
'spgw': SpgwUeContext,
's1ap': UeDescription,
'mobilityd_ipdesc_record': IPDesc,
'rules': PolicyRule,
'installed': InstalledPolicies,
}
def __init__(self):
self.client = get_default_client()
def keys(self, redis_key: str):
"""
Get current keys on redis db that match the pattern
Args:
redis_key:pattern to match the redis keys
"""
for k in self.client.keys(pattern="{}*".format(redis_key)):
deserialized_key = k.decode('utf-8')
print(deserialized_key)
def parse(self, key: str):
"""
Parse value of redis key on redis for encoded HASH, SET types, or
JSON / Protobuf encoded state-wrapped types and prints it
Args:
key: key on redis
"""
redis_type = self.client.type(key).decode('utf-8')
key_type = key
if ":" in key:
key_type = key.split(":")[1]
if redis_type == 'hash':
deserializer = self.STATE_DESERIALIZERS.get(key_type)
if not deserializer:
raise AttributeError(NO_DESERIAL_MSG.format(key_type))
self._parse_hash_type(deserializer, key)
elif redis_type == 'set':
deserializer = self.STATE_DESERIALIZERS.get(key_type)
if not deserializer:
raise AttributeError(NO_DESERIAL_MSG.format(key_type))
self._parse_set_type(deserializer, key)
else:
value = self.client.get(key)
# Try parsing as json first, if there's decoding error, parse proto
try:
self._parse_state_json(value)
except (UnicodeDecodeError, JSONDecodeError, AttributeError):
self._parse_state_proto(key_type, value)
def corrupt(self, key):
"""
Mostly used for debugging, purposely corrupts state encoded protobuf
in redis, and writes it back to datastore
Args:
key: key on redis
"""
rand_bytes = random.getrandbits(8)
byte_str = bytes([rand_bytes])
self.client[key] = byte_str
print('Corrupted %s in redis' % key)
def _parse_state_json(self, value):
if value:
deserializer = get_json_deserializer()
value = json.loads(jsonpickle.encode(deserializer(value)))
print(json.dumps(value, indent=2, sort_keys=True))
else:
raise AttributeError('Key not found on redis')
def _parse_state_proto(self, key_type, value):
proto = self.STATE_PROTOS.get(key_type.lower())
if proto:
deserializer = get_proto_deserializer(proto)
version_deserializer = get_proto_version_deserializer()
print(deserializer(value))
print('==================')
print('State version: %s' % version_deserializer(value))
else:
raise AttributeError('Key not found on redis')
def _parse_set_type(self, deserializer, key):
set_values = self.client.smembers(key)
for value in set_values:
print(deserializer(value))
def _parse_hash_type(self, deserializer, key):
value = self.client.hgetall(key)
for key, val in value.items():
print(key.decode('utf-8'))
print(deserializer(val))
def __init__(self):
serde = RedisSerde(self.REDIS_VALUE_TYPE, get_proto_serializer(),
get_proto_deserializer(ServiceExitStatus))
self._flat_dict = RedisFlatDict(get_default_client(), serde)
def __init__(self):
client = get_default_client()
super().__init__(client, self._DICT_HASH, get_proto_serializer(),
get_proto_deserializer(RedirectInformation))
def setUp(self):
self.mock_redis = fakeredis.FakeStrictRedis()
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
service = MagicMock()
service.config = {
# Replicate arbitrary orc8r protos
'state_protos': [{'proto_file': 'orc8r.protos.common_pb2',
'proto_msg': 'NetworkID',
'redis_key': NID_TYPE,
'state_scope': 'network'},
{'proto_file': 'orc8r.protos.service303_pb2',
'proto_msg': 'LogVerbosity',
'redis_key': LOG_TYPE,
'state_scope': 'gateway'},
],
'json_state': [{'redis_key': FOO_TYPE, 'state_scope': 'gateway'}]
}
service.loop = self.loop
# Bind the rpc server to a free port
self._rpc_server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10)
)
port = self._rpc_server.add_insecure_port('0.0.0.0:0')
# Add the servicer
self._servicer = DummyStateServer()
self._servicer.add_to_server(self._rpc_server)
self._rpc_server.start()
# Create a rpc stub
self.channel = grpc.insecure_channel('0.0.0.0:{}'.format(port))
serde1 = RedisSerde(NID_TYPE,
get_proto_serializer(),
get_proto_deserializer(NetworkID))
serde2 = RedisSerde(FOO_TYPE,
get_json_serializer(),
get_json_deserializer())
serde3 = RedisSerde(LOG_TYPE,
get_proto_serializer(),
get_proto_deserializer(LogVerbosity))
self.nid_client = RedisFlatDict(self.mock_redis, serde1)
self.foo_client = RedisFlatDict(self.mock_redis, serde2)
self.log_client = RedisFlatDict(self.mock_redis, serde3)
# Set up and start garbage collecting loop
grpc_client_manager = GRPCClientManager(
service_name="state",
service_stub=StateServiceStub,
max_client_reuse=60,
)
# mock the get_default_client function used to return the same
# fakeredis object
func_mock = mock.MagicMock(return_value=self.mock_redis)
with patch('magma.state.redis_dicts.get_default_client', func_mock):
# Start state garbage collection loop
self.garbage_collector = GarbageCollector(service,
grpc_client_manager)
