def __init__(self, client): serde = RedisSerde( DHCP_GW_INFO_REDIS_TYPE, get_json_serializer(), get_json_deserializer(), ) super().__init__(client, serde)
def __init__(self, client): serde = RedisSerde( ALLOCATED_IID_REDIS_TYPE, get_json_serializer(), get_json_deserializer(), ) super().__init__(client, serde)
def __init__(self): client = get_default_client() serde = RedisSerde( self._DICT_HASH, get_json_serializer(), get_json_deserializer(), ) super().__init__(client, serde, writethrough=True)
def get_json_redis_dicts(config): redis_dicts = [] json_state = config.get('json_state', []) or [] for json_cfg in json_state: is_invalid_cfg = 'redis_key' not in json_cfg or \ 'state_scope' not in json_cfg if is_invalid_cfg: logging.warning( "Invalid json state config found in json_state" "configuration: %s", json_cfg, ) continue logging.info( 'Initializing RedisSerde for json state %s', json_cfg['redis_key'], ) redis_key = json_cfg['redis_key'] serde = RedisSerde( redis_key, get_json_serializer(), get_json_deserializer(), ) redis_dict = StateDict( serde, json_cfg['state_scope'], JSON_FORMAT, ) redis_dicts.append(redis_dict) return redis_dicts
def __init__(self): client = get_default_client() super().__init__( client, self._DICT_HASH, get_json_serializer(), get_json_deserializer(), )
def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) service = MagicMock() service.config = { # Replicate arbitrary orc8r protos 'state_protos': [ { 'proto_file': 'orc8r.protos.common_pb2', 'proto_msg': 'NetworkID', 'redis_key': NID_TYPE, 'state_scope': 'network' }, { 'proto_file': 'orc8r.protos.service303_pb2', 'proto_msg': 'LogVerbosity', 'redis_key': LOG_TYPE, 'state_scope': 'gateway' }, ], 'json_state': [{ 'redis_key': FOO_TYPE, 'state_scope': 'gateway' }] } service.loop = self.loop # Bind the rpc server to a free port self._rpc_server = grpc.server( futures.ThreadPoolExecutor(max_workers=10)) port = self._rpc_server.add_insecure_port('0.0.0.0:0') # Add the servicer self._servicer = DummyStateServer() self._servicer.add_to_server(self._rpc_server) self._rpc_server.start() # Create a rpc stub self.channel = grpc.insecure_channel('0.0.0.0:{}'.format(port)) serde1 = RedisSerde(NID_TYPE, get_proto_serializer(), get_proto_deserializer(NetworkID)) serde2 = RedisSerde(FOO_TYPE, get_json_serializer(), get_json_deserializer()) serde3 = RedisSerde(LOG_TYPE, get_proto_serializer(), get_proto_deserializer(LogVerbosity)) self.nid_client = RedisFlatDict(get_default_client(), serde1) self.foo_client = RedisFlatDict(get_default_client(), serde2) self.log_client = RedisFlatDict(get_default_client(), serde3) # Set up and start garbage collecting loop grpc_client_manager = GRPCClientManager( service_name="state", service_stub=StateServiceStub, max_client_reuse=60, ) # Start state garbage collection loop self.garbage_collector = GarbageCollector(service, grpc_client_manager)
def _parse_state_json(self, value): if value: deserializer = get_json_deserializer() value = json.loads(jsonpickle.encode(deserializer(value))) print(json.dumps(value, indent=2, sort_keys=True)) else: raise AttributeError('Key not found on redis')
def __init__(self, client): serde = RedisSerde( ALLOCATED_SESSION_PREFIX_TYPE, get_json_serializer(), get_json_deserializer(), ) super().__init__(client, serde)
def __init__(self, client): serde = RedisSerde( MAC_TO_IP_REDIS_TYPE, get_json_serializer(), get_json_deserializer(), ) super().__init__(client, serde)
def __init__(self): client = get_default_client() serde = RedisSerde( ALLOCATED_SESSION_PREFIX_TYPE, get_json_serializer(), get_json_deserializer(), ) super().__init__(client, serde)
def __init__(self): client = get_default_client() serde = RedisSerde( ALLOCATED_IID_REDIS_TYPE, get_json_serializer(), get_json_deserializer(), ) super().__init__(client, serde)
def __init__(self): client = get_default_client() serde = RedisSerde( DHCP_GW_INFO_REDIS_TYPE, get_json_serializer(), get_json_deserializer(), ) super().__init__(client, serde)
def __init__(self, redis_type): self.client = get_default_client() super().__init__( self.client, redis_type, get_json_serializer(), get_json_deserializer(), )
def __init__(self, redis_type, client): self.client = client super().__init__( self.client, redis_type, get_json_serializer(), get_json_deserializer(), )
def __init__(self, print_grpc_payload: bool = False): """Initialize Directoryd grpc endpoints.""" serde = RedisSerde(DIRECTORYD_REDIS_TYPE, get_json_serializer(), get_json_deserializer()) self._redis_dict = RedisFlatDict(get_default_client(), serde) self._print_grpc_payload = print_grpc_payload if self._print_grpc_payload: logging.info("Printing GRPC messages")
def _get_json_redis_serdes(self): json_state = self._service.config.get('json_state', []) or [] for json_cfg in json_state: is_invalid_cfg = 'redis_key' not in json_cfg or \ 'state_scope' not in json_cfg if is_invalid_cfg: logging.warning( "Invalid json state config found in json_state" "configuration: %s", json_cfg) continue logging.info('Initializing RedisSerde for json state %s', json_cfg['redis_key']) redis_key = json_cfg['redis_key'] serde = StateSerde(redis_key, get_json_serializer(), get_json_deserializer(), json_cfg['state_scope'], JSON_FORMAT) self._serdes[redis_key] = serde
def _get_json_redis_clients(self): clients = [] json_state = self._service.config.get('json_state', []) or [] for json_cfg in json_state: is_invalid_cfg = 'redis_key' not in json_cfg or \ 'state_scope' not in json_cfg if is_invalid_cfg: logging.warning( "Invalid json state config found in json_state" "configuration: %s", json_cfg) continue logging.info('Initializing RedisSerde for json state %s', json_cfg['redis_key']) redis_key = json_cfg['redis_key'] serde = RedisSerde(redis_key, get_json_serializer(), get_json_deserializer()) client = StateDict(serde, json_cfg['state_scope'], JSON_FORMAT) clients.append(client) return clients
class StateCLI(object): """ CLI for debugging current Magma services state and displaying it in readable manner. """ STATE_DESERIALIZERS = { 'assigned_ip_blocks': deserialize_ip_block, 'ip_states': deserialize_ip_desc, 'sessions': _deserialize_session_json, 'rule_names': get_json_deserializer(), 'rule_ids': get_json_deserializer(), 'rule_versions': get_json_deserializer(), 'rules': get_proto_deserializer(PolicyRule), } STATE_PROTOS = { 'mme_nas_state': MmeNasState, 'spgw_state': SpgwState, 's1ap_state': S1apState, 's1ap_imsi_map': S1apImsiMap, 'mme': UeContext, 'spgw': SpgwUeContext, 's1ap': UeDescription, 'mobilityd_ipdesc_record': IPDesc, 'rules': PolicyRule, 'installed': InstalledPolicies, } def __init__(self): self.client = get_default_client() def keys(self, redis_key: str): """ Get current keys on redis db that match the pattern Args: redis_key:pattern to match the redis keys """ for k in self.client.keys(pattern="{}*".format(redis_key)): deserialized_key = k.decode('utf-8') print(deserialized_key) def parse(self, key: str): """ Parse value of redis key on redis for encoded HASH, SET types, or JSON / Protobuf encoded state-wrapped types and prints it Args: key: key on redis """ redis_type = self.client.type(key).decode('utf-8') key_type = key if ":" in key: key_type = key.split(":")[1] if redis_type == 'hash': deserializer = self.STATE_DESERIALIZERS.get(key_type) if not deserializer: raise AttributeError(NO_DESERIAL_MSG.format(key_type)) self._parse_hash_type(deserializer, key) elif redis_type == 'set': deserializer = self.STATE_DESERIALIZERS.get(key_type) if not deserializer: raise AttributeError(NO_DESERIAL_MSG.format(key_type)) self._parse_set_type(deserializer, key) else: value = self.client.get(key) # Try parsing as json first, if there's decoding error, parse proto try: self._parse_state_json(value) except (UnicodeDecodeError, JSONDecodeError, AttributeError): self._parse_state_proto(key_type, value) def corrupt(self, key): """ Mostly used for debugging, purposely corrupts state encoded protobuf in redis, and writes it back to datastore Args: key: key on redis """ rand_bytes = random.getrandbits(8) byte_str = bytes([rand_bytes]) self.client[key] = byte_str print('Corrupted %s in redis' % key) def _parse_state_json(self, value): if value: deserializer = get_json_deserializer() value = json.loads(jsonpickle.encode(deserializer(value))) print(json.dumps(value, indent=2, sort_keys=True)) else: raise AttributeError('Key not found on redis') def _parse_state_proto(self, key_type, value): proto = self.STATE_PROTOS.get(key_type.lower()) if proto: deserializer = get_proto_deserializer(proto) version_deserializer = get_proto_version_deserializer() print(deserializer(value)) print('==================') print('State version: %s' % version_deserializer(value)) else: raise AttributeError('Key not found on redis') def _parse_set_type(self, deserializer, key): set_values = self.client.smembers(key) for value in set_values: print(deserializer(value)) def _parse_hash_type(self, deserializer, key): value = self.client.hgetall(key) for key, val in value.items(): print(key.decode('utf-8')) print(deserializer(val))
def __init__(self): serde = RedisSerde(DIRECTORYD_REDIS_TYPE, get_json_serializer(), get_json_deserializer()) self._redis_dict = RedisFlatDict(get_default_client(), serde)
def __init__(self): client = get_default_client() serde = RedisSerde(MAC_TO_IP_REDIS_TYPE, get_json_serializer(), get_json_deserializer()) super().__init__(client, serde)
def setUp(self): self.mock_redis = fakeredis.FakeStrictRedis() self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) service = MagicMock() service.config = { # Replicate arbitrary orc8r protos 'state_protos': [ { 'proto_file': 'orc8r.protos.common_pb2', 'proto_msg': 'NetworkID', 'redis_key': NID_TYPE, 'state_scope': 'network', }, { 'proto_file': 'orc8r.protos.common_pb2', 'proto_msg': 'IDList', 'redis_key': IDList_TYPE, 'state_scope': 'gateway', }, { 'proto_file': 'orc8r.protos.service303_pb2', 'proto_msg': 'LogVerbosity', 'redis_key': LOG_TYPE, 'state_scope': 'gateway', }, ], 'json_state': [{ 'redis_key': FOO_TYPE, 'state_scope': 'network' }], } service.loop = self.loop # Bind the rpc server to a free port self._rpc_server = grpc.server( futures.ThreadPoolExecutor(max_workers=10), ) port = self._rpc_server.add_insecure_port('0.0.0.0:0') # Add the servicer self._servicer = DummyStateServer() self._servicer.add_to_server(self._rpc_server) self._rpc_server.start() # Create a rpc stub self.channel = grpc.insecure_channel('0.0.0.0:{}'.format(port)) serde1 = RedisSerde( NID_TYPE, get_proto_serializer(), get_proto_deserializer(NetworkID), ) serde2 = RedisSerde( IDList_TYPE, get_proto_serializer(), get_proto_deserializer(IDList), ) serde3 = RedisSerde( LOG_TYPE, get_proto_serializer(), get_proto_deserializer(LogVerbosity), ) serde4 = RedisSerde( FOO_TYPE, get_json_serializer(), get_json_deserializer(), ) self.nid_client = RedisFlatDict(self.mock_redis, serde1) self.idlist_client = RedisFlatDict(self.mock_redis, serde2) self.log_client = RedisFlatDict(self.mock_redis, serde3) self.foo_client = RedisFlatDict(self.mock_redis, serde4) # Set up and start state replicating loop grpc_client_manager = GRPCClientManager( service_name="state", service_stub=StateServiceStub, max_client_reuse=60, ) # mock the get_default_client function used to return the same # fakeredis object func_mock = mock.MagicMock(return_value=self.mock_redis) with mock.patch( 'magma.state.redis_dicts.get_default_client', func_mock, ): garbage_collector = GarbageCollector(service, grpc_client_manager) self.state_replicator = StateReplicator( service=service, garbage_collector=garbage_collector, grpc_client_manager=grpc_client_manager, ) self.state_replicator.start()