def __init__(self): client = get_default_client() super().__init__( client, self._DICT_HASH, get_proto_serializer(), get_proto_deserializer(PolicyRule), )
def __init__(self): client = get_default_client() super().__init__( client, self._DICT_HASH, get_proto_serializer(), get_proto_deserializer(ChargingRuleNameSet), )
def __init__(self): client = get_default_client() super().__init__( client, self._DICT_HASH, get_proto_serializer(), get_proto_deserializer(RatingGroup), )
def __init__(self): client = get_default_client() super().__init__( client, self._DICT_HASH, get_proto_serializer(), get_proto_deserializer(SubscriberPolicySet), ) self._clear()
def _parse_state_proto(self, key_type, value): proto = self.STATE_PROTOS.get(key_type.lower()) if proto: deserializer = get_proto_deserializer(proto) version_deserializer = get_proto_version_deserializer() print(deserializer(value)) print('==================') print('State version: %s' % version_deserializer(value)) else: raise AttributeError('Key not found on redis')
def __init__(self): client = get_default_client() super().__init__( client, self._DICT_HASH, get_proto_serializer(), get_proto_deserializer(InstalledPolicies), ) # TODO: Remove when sessiond becomes stateless self._clear()
def get_proto_redis_dicts(config): redis_dicts = [] state_protos = config.get('state_protos', []) or [] for proto_cfg in state_protos: is_invalid_cfg = 'proto_msg' not in proto_cfg or \ 'proto_file' not in proto_cfg or \ 'redis_key' not in proto_cfg or \ 'state_scope' not in proto_cfg if is_invalid_cfg: logging.warning( "Invalid proto config found in state_protos " "configuration: %s", proto_cfg, ) continue try: proto_module = importlib.import_module(proto_cfg['proto_file']) msg = getattr(proto_module, proto_cfg['proto_msg']) redis_key = proto_cfg['redis_key'] logging.info( 'Initializing RedisSerde for proto state %s', proto_cfg['redis_key'], ) serde = RedisSerde( redis_key, get_proto_serializer(), get_proto_deserializer(msg), ) redis_dict = StateDict( serde, proto_cfg['state_scope'], PROTO_FORMAT, ) redis_dicts.append(redis_dict) except (ImportError, AttributeError) as err: logging.error(err) return redis_dicts
def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) service = MagicMock() service.config = { # Replicate arbitrary orc8r protos 'state_protos': [{ 'proto_file': 'orc8r.protos.common_pb2', 'proto_msg': 'NetworkID', 'redis_key': NID_TYPE, 'state_scope': 'network' }, { 'proto_file': 'orc8r.protos.common_pb2', 'proto_msg': 'IDList', 'redis_key': IDList_TYPE, 'state_scope': 'gateway' }, { 'proto_file': 'orc8r.protos.service303_pb2', 'proto_msg': 'LogVerbosity', 'redis_key': LOG_TYPE, 'state_scope': 'gateway' }], 'json_state': [{ 'redis_key': FOO_TYPE, 'state_scope': 'network' }] } service.loop = self.loop # Bind the rpc server to a free port self._rpc_server = grpc.server( futures.ThreadPoolExecutor(max_workers=10)) port = self._rpc_server.add_insecure_port('0.0.0.0:0') # Add the servicer self._servicer = DummyStateServer() self._servicer.add_to_server(self._rpc_server) self._rpc_server.start() # Create a rpc stub self.channel = grpc.insecure_channel('0.0.0.0:{}'.format(port)) serde1 = RedisSerde(NID_TYPE, get_proto_serializer(), get_proto_deserializer(NetworkID)) serde2 = RedisSerde(IDList_TYPE, get_proto_serializer(), get_proto_deserializer(IDList)) serde3 = RedisSerde(LOG_TYPE, get_proto_serializer(), get_proto_deserializer(LogVerbosity)) serde4 = RedisSerde(FOO_TYPE, get_json_serializer(), get_json_deserializer()) self.nid_client = RedisFlatDict(get_default_client(), serde1) self.idlist_client = RedisFlatDict(get_default_client(), serde2) self.log_client = RedisFlatDict(get_default_client(), serde3) self.foo_client = RedisFlatDict(get_default_client(), serde4) # Set up and start state replicating loop grpc_client_manager = GRPCClientManager( service_name="state", service_stub=StateServiceStub, max_client_reuse=60, ) garbage_collector = GarbageCollector(service, grpc_client_manager) self.state_replicator = StateReplicator( service=service, garbage_collector=garbage_collector, grpc_client_manager=grpc_client_manager, ) self.state_replicator.start()
class StateCLI(object): """ CLI for debugging current Magma services state and displaying it in readable manner. """ STATE_DESERIALIZERS = { 'assigned_ip_blocks': deserialize_ip_block, 'ip_states': deserialize_ip_desc, 'sessions': _deserialize_session_json, 'rule_names': get_json_deserializer(), 'rule_ids': get_json_deserializer(), 'rule_versions': get_json_deserializer(), 'rules': get_proto_deserializer(PolicyRule), } STATE_PROTOS = { 'mme_nas_state': MmeNasState, 'spgw_state': SpgwState, 's1ap_state': S1apState, 's1ap_imsi_map': S1apImsiMap, 'mme': UeContext, 'spgw': SpgwUeContext, 's1ap': UeDescription, 'mobilityd_ipdesc_record': IPDesc, 'rules': PolicyRule, 'installed': InstalledPolicies, } def __init__(self): self.client = get_default_client() def keys(self, redis_key: str): """ Get current keys on redis db that match the pattern Args: redis_key:pattern to match the redis keys """ for k in self.client.keys(pattern="{}*".format(redis_key)): deserialized_key = k.decode('utf-8') print(deserialized_key) def parse(self, key: str): """ Parse value of redis key on redis for encoded HASH, SET types, or JSON / Protobuf encoded state-wrapped types and prints it Args: key: key on redis """ redis_type = self.client.type(key).decode('utf-8') key_type = key if ":" in key: key_type = key.split(":")[1] if redis_type == 'hash': deserializer = self.STATE_DESERIALIZERS.get(key_type) if not deserializer: raise AttributeError(NO_DESERIAL_MSG.format(key_type)) self._parse_hash_type(deserializer, key) elif redis_type == 'set': deserializer = self.STATE_DESERIALIZERS.get(key_type) if not deserializer: raise AttributeError(NO_DESERIAL_MSG.format(key_type)) self._parse_set_type(deserializer, key) else: value = self.client.get(key) # Try parsing as json first, if there's decoding error, parse proto try: self._parse_state_json(value) except (UnicodeDecodeError, JSONDecodeError, AttributeError): self._parse_state_proto(key_type, value) def corrupt(self, key): """ Mostly used for debugging, purposely corrupts state encoded protobuf in redis, and writes it back to datastore Args: key: key on redis """ rand_bytes = random.getrandbits(8) byte_str = bytes([rand_bytes]) self.client[key] = byte_str print('Corrupted %s in redis' % key) def _parse_state_json(self, value): if value: deserializer = get_json_deserializer() value = json.loads(jsonpickle.encode(deserializer(value))) print(json.dumps(value, indent=2, sort_keys=True)) else: raise AttributeError('Key not found on redis') def _parse_state_proto(self, key_type, value): proto = self.STATE_PROTOS.get(key_type.lower()) if proto: deserializer = get_proto_deserializer(proto) version_deserializer = get_proto_version_deserializer() print(deserializer(value)) print('==================') print('State version: %s' % version_deserializer(value)) else: raise AttributeError('Key not found on redis') def _parse_set_type(self, deserializer, key): set_values = self.client.smembers(key) for value in set_values: print(deserializer(value)) def _parse_hash_type(self, deserializer, key): value = self.client.hgetall(key) for key, val in value.items(): print(key.decode('utf-8')) print(deserializer(val))
def __init__(self): serde = RedisSerde(self.REDIS_VALUE_TYPE, get_proto_serializer(), get_proto_deserializer(ServiceExitStatus)) self._flat_dict = RedisFlatDict(get_default_client(), serde)
def __init__(self): client = get_default_client() super().__init__(client, self._DICT_HASH, get_proto_serializer(), get_proto_deserializer(RedirectInformation))
def setUp(self): self.mock_redis = fakeredis.FakeStrictRedis() self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) service = MagicMock() service.config = { # Replicate arbitrary orc8r protos 'state_protos': [{'proto_file': 'orc8r.protos.common_pb2', 'proto_msg': 'NetworkID', 'redis_key': NID_TYPE, 'state_scope': 'network'}, {'proto_file': 'orc8r.protos.service303_pb2', 'proto_msg': 'LogVerbosity', 'redis_key': LOG_TYPE, 'state_scope': 'gateway'}, ], 'json_state': [{'redis_key': FOO_TYPE, 'state_scope': 'gateway'}] } service.loop = self.loop # Bind the rpc server to a free port self._rpc_server = grpc.server( futures.ThreadPoolExecutor(max_workers=10) ) port = self._rpc_server.add_insecure_port('0.0.0.0:0') # Add the servicer self._servicer = DummyStateServer() self._servicer.add_to_server(self._rpc_server) self._rpc_server.start() # Create a rpc stub self.channel = grpc.insecure_channel('0.0.0.0:{}'.format(port)) serde1 = RedisSerde(NID_TYPE, get_proto_serializer(), get_proto_deserializer(NetworkID)) serde2 = RedisSerde(FOO_TYPE, get_json_serializer(), get_json_deserializer()) serde3 = RedisSerde(LOG_TYPE, get_proto_serializer(), get_proto_deserializer(LogVerbosity)) self.nid_client = RedisFlatDict(self.mock_redis, serde1) self.foo_client = RedisFlatDict(self.mock_redis, serde2) self.log_client = RedisFlatDict(self.mock_redis, serde3) # Set up and start garbage collecting loop grpc_client_manager = GRPCClientManager( service_name="state", service_stub=StateServiceStub, max_client_reuse=60, ) # mock the get_default_client function used to return the same # fakeredis object func_mock = mock.MagicMock(return_value=self.mock_redis) with patch('magma.state.redis_dicts.get_default_client', func_mock): # Start state garbage collection loop self.garbage_collector = GarbageCollector(service, grpc_client_manager)