def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
service = MagicMock()
service.config = {
# Replicate arbitrary orc8r protos
'state_protos': [
{
'proto_file': 'orc8r.protos.common_pb2',
'proto_msg': 'NetworkID',
'redis_key': NID_TYPE,
'state_scope': 'network'
},
{
'proto_file': 'orc8r.protos.service303_pb2',
'proto_msg': 'LogVerbosity',
'redis_key': LOG_TYPE,
'state_scope': 'gateway'
},
],
'json_state': [{
'redis_key': FOO_TYPE,
'state_scope': 'gateway'
}]
}
service.loop = self.loop
# Bind the rpc server to a free port
self._rpc_server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10))
port = self._rpc_server.add_insecure_port('0.0.0.0:0')
# Add the servicer
self._servicer = DummyStateServer()
self._servicer.add_to_server(self._rpc_server)
self._rpc_server.start()
# Create a rpc stub
self.channel = grpc.insecure_channel('0.0.0.0:{}'.format(port))
serde1 = RedisSerde(NID_TYPE, get_proto_serializer(),
get_proto_deserializer(NetworkID))
serde2 = RedisSerde(FOO_TYPE, get_json_serializer(),
get_json_deserializer())
serde3 = RedisSerde(LOG_TYPE, get_proto_serializer(),
get_proto_deserializer(LogVerbosity))
self.nid_client = RedisFlatDict(get_default_client(), serde1)
self.foo_client = RedisFlatDict(get_default_client(), serde2)
self.log_client = RedisFlatDict(get_default_client(), serde3)
# Set up and start garbage collecting loop
grpc_client_manager = GRPCClientManager(
service_name="state",
service_stub=StateServiceStub,
max_client_reuse=60,
)
# Start state garbage collection loop
self.garbage_collector = GarbageCollector(service, grpc_client_manager)
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_json_serializer(), get_json_deserializer(),
)
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_proto_serializer(),
get_proto_deserializer(PolicyRule))
def __init__(self):
serde = RedisSerde(
self.REDIS_VALUE_TYPE,
get_proto_serializer(),
get_proto_deserializer(ServiceExitStatus),
)
self._flat_dict = RedisFlatDict(get_default_client(), serde)
def _new_ip_allocator(self, recycling_interval):
"""
Creates and sets up an IPAllocator with the given recycling interval.
"""
store = MobilityStore(get_default_client(), False, 3980)
ip_allocator = IpAllocatorPool(store)
ip_allocator_static = IPAllocatorStaticWrapper(
store,
MockedSubscriberDBStub(),
ip_allocator,
)
ipv4_allocator = IPAllocatorMultiAPNWrapper(
store,
subscriberdb_rpc_stub=MockedSubscriberDBStub(),
ip_allocator=ip_allocator_static,
)
ipv6_allocator = IPv6AllocatorPool(
store,
session_prefix_alloc_mode='RANDOM',
)
self._allocator = IPAddressManager(
ipv4_allocator,
ipv6_allocator,
store,
recycling_interval,
)
self._allocator.add_ip_block(self._block)
def __init__(self):
client = get_default_client()
serde = RedisSerde(
self._DICT_HASH, get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde, writethrough=True)
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_proto_serializer(),
get_proto_deserializer(ChargingRuleNameSet))
def __init__(self,
*,
config,
recycling_interval: int = DEFAULT_IP_RECYCLE_INTERVAL):
""" Initializes a new IP allocator
Args:
recycling_interval (number): minimum time, in seconds, before a
released IP is recycled and freed into the pool of available
IPs.
Default: None, no recycling will occur automatically.
"""
persist_to_redis = config.get('persist_to_redis', True)
redis_port = config.get('redis_port', 6379)
if 'allocator_type' in config:
if config['allocator_type'] == 'ip_pool':
self.allocator_type = IPAllocatorType.IP_POOL
elif config['allocator_type'] == 'dhcp':
self.allocator_type = IPAllocatorType.DHCP
else:
raise ArgumentError("unknown allocator config {}"
.format(config['allocator_type']))
else:
self.allocator_type = IPAllocatorType.IP_POOL
logging.debug('Persist to Redis: %s', persist_to_redis)
self._lock = threading.RLock() # re-entrant locks
self._recycle_timer = None # reference to recycle timer
self._recycling_interval_seconds = recycling_interval
if not persist_to_redis:
self._assigned_ip_blocks = set() # {ip_block}
self.sid_ips_map = defaultdict(IPDesc) # {SID=>IPDesc}
else:
if not redis_port:
raise ValueError(
'Must specify a redis_port in mobilityd config.')
client = get_default_client()
self._assigned_ip_blocks = store.AssignedIpBlocksSet(client)
self.sid_ips_map = store.IPDescDict(client)
self.ip_state_map = IpDescriptorMap(persist_to_redis, redis_port)
if self.allocator_type == IPAllocatorType.IP_POOL:
self.ip_allocator = IpAllocatorStatic(self._assigned_ip_blocks,
self.ip_state_map,
self.sid_ips_map)
elif self.allocator_type == IPAllocatorType.DHCP:
dhcp_store = store.MacToIP() # mac => DHCP_State
dhcp_gw_info = UplinkGatewayInfo(store.GatewayInfoMap())
iface = config.get('dhcp_iface', 'dhcp0')
retry_limit = config.get('retry_limit', 300)
self.ip_allocator = IPAllocatorDHCP(self._assigned_ip_blocks,
self.ip_state_map,
iface=iface,
retry_limit=retry_limit,
dhcp_store=dhcp_store,
gw_info=dhcp_gw_info)
def clear_redis_state():
if os.getuid() != 0:
print("Need to run as root to clear Redis state.")
sys.exit(return_codes.INVALID)
# stop MME, which in turn stops mobilityd, pipelined and sessiond
subprocess.call("service magma@mme stop".split())
# delete all keys from Redis which capture service state
redis_client = get_default_client()
for key_regex in [
"*_state",
"IMSI*MME",
"IMSI*S1AP",
"IMSI*SPGW",
"IMSI*mobilityd*",
"mobilityd:assigned_ip_blocks",
"mobilityd:ip_states:*",
"NO_VLAN:mobilityd_gw_info",
"QosManager",
"s1ap_imsi_map",
"sessiond:sessions",
"*pipelined:rule_ids",
"*pipelined:rule_versions",
"*pipelined:rule_names",
"mme_ueip_imsi_map",
]:
for key in redis_client.scan_iter(key_regex):
redis_client.delete(key)
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_proto_serializer(),
get_proto_deserializer(RedirectInformation))
def setUp(self):
self._br = "t_up_br0"
setup_dhcp_server = SCRIPT_PATH + "scripts/setup-test-dhcp-srv.sh"
subprocess.check_call([setup_dhcp_server, "t0"])
setup_uplink_br = [
SCRIPT_PATH + "scripts/setup-uplink-br.sh",
self._br,
"t0uplink_p0",
"t0_dhcp1",
]
subprocess.check_call(setup_uplink_br)
store = MobilityStore(get_default_client(), False, 3980)
ipv4_allocator = IPAllocatorDHCP(
store,
iface='t0uplink_p0',
retry_limit=50,
)
ipv6_allocator = IPv6AllocatorPool(
store,
session_prefix_alloc_mode='RANDOM',
)
self._dhcp_allocator = IPAddressManager(
ipv4_allocator,
ipv6_allocator,
store,
recycling_interval=2,
)
def debug(cls, _, __, ___):
config = load_service_config('pipelined')
qos_impl_type = QosImplType(config["qos"]["impl"])
qos_store = QosStore(cls.__name__, client=get_default_client())
for k, v in qos_store.items():
_, imsi, ip_addr, rule_num, d = get_key(k)
_, qid, ambr, leaf = get_data(v)
print('imsi :', imsi)
print('ip_addr :', ip_addr)
print('rule_num :', rule_num)
print('direction :', d)
print('qos_handle:', qid)
print('qos_handle_ambr:', ambr)
print('qos_handle_ambr_leaf:', leaf)
if qos_impl_type == QosImplType.OVS_METER:
MeterManager.dump_meter_state(v)
else:
dev = config[
'nat_iface'] if d == FlowMatch.UPLINK else 'gtpu_sys_2152'
print("Dev: ", dev)
TrafficClass.dump_class_state(dev, qid)
if leaf and leaf != qid:
print("Leaf:")
TrafficClass.dump_class_state(dev, leaf)
if ambr:
print("AMBR (parent):")
TrafficClass.dump_class_state(dev, ambr)
if qos_impl_type == QosImplType.LINUX_TC:
dev = config['nat_iface']
print("Root stats for: ", dev)
TrafficClass.dump_root_class_stats(dev)
dev = 'gtpu_sys_2152'
print("Root stats for: ", dev)
TrafficClass.dump_root_class_stats(dev)
def __init__(self):
client = get_default_client()
serde = RedisSerde(
ALLOCATED_IID_REDIS_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde)
def __init__(self):
client = get_default_client()
serde = RedisSerde(
DHCP_GW_INFO_REDIS_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde)
def __init__(self, redis_type):
self.client = get_default_client()
super().__init__(
self.client,
redis_type,
get_json_serializer(),
get_json_deserializer(),
)
def __init__(self):
client = get_default_client()
serde = RedisSerde(
ALLOCATED_SESSION_PREFIX_TYPE,
get_json_serializer(),
get_json_deserializer(),
)
super().__init__(client, serde)
def _new_ip_allocator(self, block):
"""
Creates and sets up an IPAllocator with the given IPv6 block.
"""
store = MobilityStore(get_default_client(), False, 3980)
allocator = IPv6AllocatorPool(store, 'RANDOM')
allocator.add_ip_block(block)
return allocator
def __init__(self, print_grpc_payload: bool = False):
"""Initialize Directoryd grpc endpoints."""
serde = RedisSerde(DIRECTORYD_REDIS_TYPE, get_json_serializer(),
get_json_deserializer())
self._redis_dict = RedisFlatDict(get_default_client(), serde)
self._print_grpc_payload = print_grpc_payload
if self._print_grpc_payload:
logging.info("Printing GRPC messages")
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_proto_serializer(),
get_proto_deserializer(SubscriberPolicySet),
)
self._clear()
def setUp(self):
client = get_default_client()
# Use arbitrary orc8r proto to test with
self._hash_dict = RedisHashDict(client, "unittest",
get_proto_serializer(),
get_proto_deserializer(LogVerbosity))
serde = RedisSerde('log_verbosity', get_proto_serializer(),
get_proto_deserializer(LogVerbosity))
self._flat_dict = RedisFlatDict(client, serde)
def __init__(self):
client = get_default_client()
super().__init__(
client,
self._DICT_HASH,
get_proto_serializer(),
get_proto_deserializer(InstalledPolicies),
)
# TODO: Remove when sessiond becomes stateless
self._clear()
def _new_ip_allocator(self, recycling_interval):
"""
Creates and sets up an IPAllocator with the given recycling interval.
"""
store = MobilityStore(get_default_client(), False, 3980)
store.dhcp_gw_info.read_default_gw()
ip_allocator = IpAllocatorPool(store)
ipv6_allocator = IPv6AllocatorPool(store,
session_prefix_alloc_mode='RANDOM')
self._allocator = IPAddressManager(ip_allocator, ipv6_allocator, store,
recycling_interval)
self._allocator.add_ip_block(self._block)
def setUp(self):
# Bind the rpc server to a free port
thread_pool = futures.ThreadPoolExecutor(max_workers=10)
self._rpc_server = grpc.server(thread_pool)
port = self._rpc_server.add_insecure_port('0.0.0.0:0')
store = MobilityStore(get_default_client(), False, 3980)
store.dhcp_gw_info.read_default_gw()
ip_allocator = IpAllocatorPool(store)
ipv6_allocator = IPv6AllocatorPool(
store,
session_prefix_alloc_mode='RANDOM',
)
self._allocator = IPAddressManager(
ip_allocator,
ipv6_allocator,
store,
)
# Add the servicer
self._servicer = MobilityServiceRpcServicer(self._allocator, False)
self._servicer.add_to_server(self._rpc_server)
self._rpc_server.start()
# Create a rpc stub
channel = grpc.insecure_channel('0.0.0.0:{}'.format(port))
self._stub = MobilityServiceStub(channel)
# variables shared across tests
self._netaddr = '192.168.0.0'
self._prefix_len = 28
ip_bytes = bytes(map(int, self._netaddr.split('.')))
self._block_msg = IPBlock(
version=IPBlock.IPV4,
net_address=ip_bytes,
prefix_len=self._prefix_len,
)
self._ipv6_block = ipaddress.ip_network('fdee:5:6c::/48')
self._ipv6_netaddr = self._ipv6_block.network_address.packed
self._ipv6_block_msg = IPBlock(
version=IPBlock.IPV6,
net_address=self._ipv6_netaddr,
prefix_len=self._ipv6_block.prefixlen,
)
self._block = ipaddress.ip_network(
"%s/%s" % (self._netaddr, self._prefix_len),
)
self._sid0 = SIDUtils.to_pb('IMSI0')
self._sid1 = SIDUtils.to_pb('IMSI1')
self._sid2 = SIDUtils.to_pb('IMSI2')
self._apn0 = 'Internet'
self._apn1 = 'IMS'
def main():
""" main() for MobilityD """
service = MagmaService('mobilityd', mconfigs_pb2.MobilityD())
# Load service configs and mconfig
config = service.config
mconfig = service.mconfig
multi_apn = config.get('multi_apn', mconfig.multi_apn_ip_alloc)
static_ip_enabled = config.get('static_ip', mconfig.static_ip_enabled)
allocator_type = mconfig.ip_allocator_type
dhcp_iface = config.get('dhcp_iface', 'dhcp0')
dhcp_retry_limit = config.get('retry_limit', 300)
# TODO: consider adding gateway mconfig to decide whether to
# persist to Redis
client = get_default_client()
store = MobilityStore(client, config.get('persist_to_redis', False),
config.get('redis_port', 6380))
chan = ServiceRegistry.get_rpc_channel('subscriberdb',
ServiceRegistry.LOCAL)
ipv4_allocator = _get_ipv4_allocator(store, allocator_type,
static_ip_enabled, multi_apn,
dhcp_iface, dhcp_retry_limit,
SubscriberDBStub(chan))
# Init IPv6 allocator, for now only IP_POOL mode is supported for IPv6
ipv6_allocation_type = config['ipv6_ip_allocator_type']
ipv6_allocator = IPv6AllocatorPool(
store=store, session_prefix_alloc_mode=ipv6_allocation_type)
# Load IPAddressManager
ip_address_man = IPAddressManager(ipv4_allocator, ipv6_allocator, store)
# Add all servicers to the server
mobility_service_servicer = MobilityServiceRpcServicer(
ip_address_man, mconfig.ip_block, config.get('ipv6_prefix_block'))
mobility_service_servicer.add_to_server(service.rpc_server)
# Run the service loop
service.run()
# Cleanup the service
service.close()
def __init__(self, service: MagmaService,
grpc_client_manager: GRPCClientManager):
super().__init__(DEFAULT_SYNC_INTERVAL, service.loop)
self._service = service
# In memory mapping of states to version
self._state_versions = {}
# Serdes for each type of state to replicate
self._serdes = {}
self._get_proto_redis_serdes()
self._get_json_redis_serdes()
self._redis_client = RedisFlatDict(get_default_client(), self._serdes)
# _grpc_client_manager to manage grpc client recyclings
self._grpc_client_manager = grpc_client_manager
# Flag to indicate if resync has completed successfully.
# Replication cannot proceed until this flag is True
self._has_resync_completed = False
def __init__(self, loop, config, client=get_default_client()):
self._initialized = False
self._clean_restart = config["clean_restart"]
self._subscriber_state = {}
self._loop = loop
self._redis_conn_retry_secs = 1
self._config = config
# protect QoS object create and delete across a QoSManager Object.
self._lock = threading.Lock()
if 'qos' not in config.keys():
LOG.error("qos field not provided in config")
return
self._qos_enabled = config["qos"]["enable"]
if not self._qos_enabled:
return
self._apn_ambr_enabled = config["qos"].get("apn_ambr_enabled", True)
LOG.info("QoS: apn_ambr_enabled: %s", self._apn_ambr_enabled)
self._redis_store = QosStore(self.__class__.__name__, client)
self.impl = None
def main():
""" main() for MobilityD """
service = MagmaService('mobilityd', mconfigs_pb2.MobilityD())
# Load service configs and mconfig
config = service.config
mconfig = service.mconfig
multi_apn = config.get('multi_apn', mconfig.multi_apn_ip_alloc)
static_ip_enabled = config.get('static_ip', mconfig.static_ip_enabled)
allocator_type = mconfig.ip_allocator_type
dhcp_iface = config.get('dhcp_iface', 'dhcp0')
dhcp_retry_limit = config.get('retry_limit', 300)
# TODO: consider adding gateway mconfig to decide whether to
# persist to Redis
client = get_default_client()
store = MobilityStore(client, config.get('persist_to_redis', False),
config.get('redis_port', 6380))
chan = ServiceRegistry.get_rpc_channel('subscriberdb',
ServiceRegistry.LOCAL)
ipv4_allocator = _get_ipv4_allocator(store, allocator_type,
static_ip_enabled, multi_apn,
dhcp_iface, dhcp_retry_limit,
SubscriberDBStub(chan))
# Init IPv6 allocator, for now only IP_POOL mode is supported for IPv6
ipv6_prefix_allocation_type = mconfig.ipv6_prefix_allocation_type or \
DEFAULT_IPV6_PREFIX_ALLOC_MODE
ipv6_allocator = IPv6AllocatorPool(
store=store, session_prefix_alloc_mode=ipv6_prefix_allocation_type)
# Load IPAddressManager
ip_address_man = IPAddressManager(ipv4_allocator, ipv6_allocator, store)
# Add all servicers to the server
mobility_service_servicer = MobilityServiceRpcServicer(
ip_address_man, config.get('print_grpc_payload', False))
mobility_service_servicer.add_to_server(service.rpc_server)
# Load IPv4 and IPv6 blocks from the configurable mconfig file
# No dynamic reloading support for now, assume restart after updates
logging.info('Adding IPv4 block')
ipv4_block = _get_ip_block(mconfig.ip_block)
if ipv4_block is not None:
try:
mobility_service_servicer.add_ip_block(ipv4_block)
except OverlappedIPBlocksError:
logging.warning("Overlapped IPv4 block: %s", ipv4_block)
logging.info('Adding IPv6 block')
ipv6_block = _get_ip_block(mconfig.ipv6_block)
if ipv6_block is not None:
try:
mobility_service_servicer.add_ip_block(ipv6_block)
except OverlappedIPBlocksError:
logging.warning("Overlapped IPv6 block: %s", ipv6_block)
# Run the service loop
service.run()
# Cleanup the service
service.close()
def __init__(self, serde: RedisSerde, state_scope: str, state_format: int):
super().__init__(get_default_client(), serde)
# Scope determines the deviceID to report the state with
self.state_scope = state_scope
self.state_format = state_format