def __init__(self, client):
serde = RedisSerde(
DHCP_GW_INFO_REDIS_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde)
def __init__(self, client):
serde = RedisSerde(
ALLOCATED_IID_REDIS_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde)
def __init__(self):
client = get_default_client()
serde = RedisSerde(
self._DICT_HASH, get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde, writethrough=True)
def get_json_redis_dicts(config):
redis_dicts = []
json_state = config.get('json_state', []) or []
for json_cfg in json_state:
is_invalid_cfg = 'redis_key' not in json_cfg or \
'state_scope' not in json_cfg
if is_invalid_cfg:
logging.warning(
"Invalid json state config found in json_state"
"configuration: %s",
json_cfg,
)
continue
logging.info(
'Initializing RedisSerde for json state %s',
json_cfg['redis_key'],
)
redis_key = json_cfg['redis_key']
serde = RedisSerde(
redis_key,
get_json_serializer(),
get_json_deserializer(),
)
redis_dict = StateDict(
serde,
json_cfg['state_scope'],
JSON_FORMAT,
)
redis_dicts.append(redis_dict)
return redis_dicts
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_json_serializer(), get_json_deserializer(),
)
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
service = MagicMock()
service.config = {
# Replicate arbitrary orc8r protos
'state_protos': [
{
'proto_file': 'orc8r.protos.common_pb2',
'proto_msg': 'NetworkID',
'redis_key': NID_TYPE,
'state_scope': 'network'
},
{
'proto_file': 'orc8r.protos.service303_pb2',
'proto_msg': 'LogVerbosity',
'redis_key': LOG_TYPE,
'state_scope': 'gateway'
},
],
'json_state': [{
'redis_key': FOO_TYPE,
'state_scope': 'gateway'
}]
}
service.loop = self.loop
# Bind the rpc server to a free port
self._rpc_server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10))
port = self._rpc_server.add_insecure_port('0.0.0.0:0')
# Add the servicer
self._servicer = DummyStateServer()
self._servicer.add_to_server(self._rpc_server)
self._rpc_server.start()
# Create a rpc stub
self.channel = grpc.insecure_channel('0.0.0.0:{}'.format(port))
serde1 = RedisSerde(NID_TYPE, get_proto_serializer(),
get_proto_deserializer(NetworkID))
serde2 = RedisSerde(FOO_TYPE, get_json_serializer(),
get_json_deserializer())
serde3 = RedisSerde(LOG_TYPE, get_proto_serializer(),
get_proto_deserializer(LogVerbosity))
self.nid_client = RedisFlatDict(get_default_client(), serde1)
self.foo_client = RedisFlatDict(get_default_client(), serde2)
self.log_client = RedisFlatDict(get_default_client(), serde3)
# Set up and start garbage collecting loop
grpc_client_manager = GRPCClientManager(
service_name="state",
service_stub=StateServiceStub,
max_client_reuse=60,
)
# Start state garbage collection loop
self.garbage_collector = GarbageCollector(service, grpc_client_manager)
def _parse_state_json(self, value):
if value:
deserializer = get_json_deserializer()
value = json.loads(jsonpickle.encode(deserializer(value)))
print(json.dumps(value, indent=2, sort_keys=True))
else:
raise AttributeError('Key not found on redis')
def __init__(self, client):
serde = RedisSerde(
ALLOCATED_SESSION_PREFIX_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde)
def __init__(self, client):
serde = RedisSerde(
MAC_TO_IP_REDIS_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde)
def __init__(self):
client = get_default_client()
serde = RedisSerde(
ALLOCATED_SESSION_PREFIX_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde)
def __init__(self):
client = get_default_client()
serde = RedisSerde(
ALLOCATED_IID_REDIS_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde)
def __init__(self):
client = get_default_client()
serde = RedisSerde(
DHCP_GW_INFO_REDIS_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde)
def __init__(self, redis_type):
self.client = get_default_client()
super().__init__(
self.client,
redis_type,
get_json_serializer(),
get_json_deserializer(),
)
def __init__(self, redis_type, client):
self.client = client
super().__init__(
self.client,
redis_type,
get_json_serializer(),
get_json_deserializer(),
)
def __init__(self, print_grpc_payload: bool = False):
"""Initialize Directoryd grpc endpoints."""
serde = RedisSerde(DIRECTORYD_REDIS_TYPE, get_json_serializer(),
get_json_deserializer())
self._redis_dict = RedisFlatDict(get_default_client(), serde)
self._print_grpc_payload = print_grpc_payload
if self._print_grpc_payload:
logging.info("Printing GRPC messages")
def _get_json_redis_serdes(self):
json_state = self._service.config.get('json_state', []) or []
for json_cfg in json_state:
is_invalid_cfg = 'redis_key' not in json_cfg or \
'state_scope' not in json_cfg
if is_invalid_cfg:
logging.warning(
"Invalid json state config found in json_state"
"configuration: %s", json_cfg)
continue
logging.info('Initializing RedisSerde for json state %s',
json_cfg['redis_key'])
redis_key = json_cfg['redis_key']
serde = StateSerde(redis_key, get_json_serializer(),
get_json_deserializer(),
json_cfg['state_scope'], JSON_FORMAT)
self._serdes[redis_key] = serde
def _get_json_redis_clients(self):
clients = []
json_state = self._service.config.get('json_state', []) or []
for json_cfg in json_state:
is_invalid_cfg = 'redis_key' not in json_cfg or \
'state_scope' not in json_cfg
if is_invalid_cfg:
logging.warning(
"Invalid json state config found in json_state"
"configuration: %s", json_cfg)
continue
logging.info('Initializing RedisSerde for json state %s',
json_cfg['redis_key'])
redis_key = json_cfg['redis_key']
serde = RedisSerde(redis_key, get_json_serializer(),
get_json_deserializer())
client = StateDict(serde, json_cfg['state_scope'], JSON_FORMAT)
clients.append(client)
return clients
class StateCLI(object):
"""
CLI for debugging current Magma services state and displaying it
in readable manner.
"""
STATE_DESERIALIZERS = {
'assigned_ip_blocks': deserialize_ip_block,
'ip_states': deserialize_ip_desc,
'sessions': _deserialize_session_json,
'rule_names': get_json_deserializer(),
'rule_ids': get_json_deserializer(),
'rule_versions': get_json_deserializer(),
'rules': get_proto_deserializer(PolicyRule),
}
STATE_PROTOS = {
'mme_nas_state': MmeNasState,
'spgw_state': SpgwState,
's1ap_state': S1apState,
's1ap_imsi_map': S1apImsiMap,
'mme': UeContext,
'spgw': SpgwUeContext,
's1ap': UeDescription,
'mobilityd_ipdesc_record': IPDesc,
'rules': PolicyRule,
'installed': InstalledPolicies,
}
def __init__(self):
self.client = get_default_client()
def keys(self, redis_key: str):
"""
Get current keys on redis db that match the pattern
Args:
redis_key:pattern to match the redis keys
"""
for k in self.client.keys(pattern="{}*".format(redis_key)):
deserialized_key = k.decode('utf-8')
print(deserialized_key)
def parse(self, key: str):
"""
Parse value of redis key on redis for encoded HASH, SET types, or
JSON / Protobuf encoded state-wrapped types and prints it
Args:
key: key on redis
"""
redis_type = self.client.type(key).decode('utf-8')
key_type = key
if ":" in key:
key_type = key.split(":")[1]
if redis_type == 'hash':
deserializer = self.STATE_DESERIALIZERS.get(key_type)
if not deserializer:
raise AttributeError(NO_DESERIAL_MSG.format(key_type))
self._parse_hash_type(deserializer, key)
elif redis_type == 'set':
deserializer = self.STATE_DESERIALIZERS.get(key_type)
if not deserializer:
raise AttributeError(NO_DESERIAL_MSG.format(key_type))
self._parse_set_type(deserializer, key)
else:
value = self.client.get(key)
# Try parsing as json first, if there's decoding error, parse proto
try:
self._parse_state_json(value)
except (UnicodeDecodeError, JSONDecodeError, AttributeError):
self._parse_state_proto(key_type, value)
def corrupt(self, key):
"""
Mostly used for debugging, purposely corrupts state encoded protobuf
in redis, and writes it back to datastore
Args:
key: key on redis
"""
rand_bytes = random.getrandbits(8)
byte_str = bytes([rand_bytes])
self.client[key] = byte_str
print('Corrupted %s in redis' % key)
def _parse_state_json(self, value):
if value:
deserializer = get_json_deserializer()
value = json.loads(jsonpickle.encode(deserializer(value)))
print(json.dumps(value, indent=2, sort_keys=True))
else:
raise AttributeError('Key not found on redis')
def _parse_state_proto(self, key_type, value):
proto = self.STATE_PROTOS.get(key_type.lower())
if proto:
deserializer = get_proto_deserializer(proto)
version_deserializer = get_proto_version_deserializer()
print(deserializer(value))
print('==================')
print('State version: %s' % version_deserializer(value))
else:
raise AttributeError('Key not found on redis')
def _parse_set_type(self, deserializer, key):
set_values = self.client.smembers(key)
for value in set_values:
print(deserializer(value))
def _parse_hash_type(self, deserializer, key):
value = self.client.hgetall(key)
for key, val in value.items():
print(key.decode('utf-8'))
print(deserializer(val))
def __init__(self):
serde = RedisSerde(DIRECTORYD_REDIS_TYPE, get_json_serializer(),
get_json_deserializer())
self._redis_dict = RedisFlatDict(get_default_client(), serde)
def __init__(self):
client = get_default_client()
serde = RedisSerde(MAC_TO_IP_REDIS_TYPE,
get_json_serializer(), get_json_deserializer())
super().__init__(client, serde)
def setUp(self):
self.mock_redis = fakeredis.FakeStrictRedis()
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
service = MagicMock()
service.config = {
# Replicate arbitrary orc8r protos
'state_protos': [
{
'proto_file': 'orc8r.protos.common_pb2',
'proto_msg': 'NetworkID',
'redis_key': NID_TYPE,
'state_scope': 'network',
},
{
'proto_file': 'orc8r.protos.common_pb2',
'proto_msg': 'IDList',
'redis_key': IDList_TYPE,
'state_scope': 'gateway',
},
{
'proto_file': 'orc8r.protos.service303_pb2',
'proto_msg': 'LogVerbosity',
'redis_key': LOG_TYPE,
'state_scope': 'gateway',
},
],
'json_state': [{
'redis_key': FOO_TYPE,
'state_scope': 'network'
}],
}
service.loop = self.loop
# Bind the rpc server to a free port
self._rpc_server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10), )
port = self._rpc_server.add_insecure_port('0.0.0.0:0')
# Add the servicer
self._servicer = DummyStateServer()
self._servicer.add_to_server(self._rpc_server)
self._rpc_server.start()
# Create a rpc stub
self.channel = grpc.insecure_channel('0.0.0.0:{}'.format(port))
serde1 = RedisSerde(
NID_TYPE,
get_proto_serializer(),
get_proto_deserializer(NetworkID),
)
serde2 = RedisSerde(
IDList_TYPE,
get_proto_serializer(),
get_proto_deserializer(IDList),
)
serde3 = RedisSerde(
LOG_TYPE,
get_proto_serializer(),
get_proto_deserializer(LogVerbosity),
)
serde4 = RedisSerde(
FOO_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
self.nid_client = RedisFlatDict(self.mock_redis, serde1)
self.idlist_client = RedisFlatDict(self.mock_redis, serde2)
self.log_client = RedisFlatDict(self.mock_redis, serde3)
self.foo_client = RedisFlatDict(self.mock_redis, serde4)
# Set up and start state replicating loop
grpc_client_manager = GRPCClientManager(
service_name="state",
service_stub=StateServiceStub,
max_client_reuse=60,
)
# mock the get_default_client function used to return the same
# fakeredis object
func_mock = mock.MagicMock(return_value=self.mock_redis)
with mock.patch(
'magma.state.redis_dicts.get_default_client',
func_mock,
):
garbage_collector = GarbageCollector(service, grpc_client_manager)
self.state_replicator = StateReplicator(
service=service,
garbage_collector=garbage_collector,
grpc_client_manager=grpc_client_manager,
)
self.state_replicator.start()
