def filter_configs_by_key(configs_by_key: Dict[str, TAny]) -> Dict[str, TAny]: """ Given a JSON-deserialized map of mconfig protobuf Any's keyed by service name, filter out any entires without a corresponding service or which have values that aren't registered in the protobuf symbol database yet. Args: configs_by_key: JSON-deserialized service mconfigs keyed by service name Returns: The input map without any services which currently don't exist. """ magmad_cfg = service_configs.load_service_config('magmad') services = magmad_cfg.get('magma_services', []) services.append('magmad') services += magmad_cfg.get('registered_dynamic_services', []) services = set(services) filtered_configs_by_key = {} for srv, cfg in configs_by_key.items(): if srv not in services: continue filtered_configs_by_key[srv] = cfg return filtered_configs_by_key
def _check_stateless_service_config(service, config_name, config_value): service_config = load_service_config(service) if service_config.get(config_name) == config_value: return return_codes.STATELESS print(service, "is stateful") return return_codes.STATEFUL
def get_context(): """ Provide context to pass to Jinja2 for templating. """ context = {} cfg = load_service_config("dnsd") try: mconfig = load_service_mconfig('dnsd', DnsD()) except LoadConfigError as err: logging.warning("Error! Using default config because: %s", err) mconfig = DnsD() ip = get_ip_from_if_cidr(cfg['enodeb_interface']) if int(ip.split('/')[1]) < 16: logging.fatal("Large interface netmasks hang dnsmasq, consider using a " "netmask in range /16 - /24") raise Exception("Interface %s netmask is to large." % cfg['enodeb_interface']) dhcp_block_size = cfg['dhcp_block_size'] available_hosts = list(ipaddress.IPv4Interface(ip).network.hosts()) context['dhcp_server_enabled'] = mconfig.dhcp_server_enabled if dhcp_block_size < len(available_hosts): context['dhcp_range'] = { "lower": str(available_hosts[-dhcp_block_size]), "upper": str(available_hosts[-1]), } else: logging.fatal("Not enough available hosts to allocate a DHCP block of \ %d addresses." % (dhcp_block_size)) context['addresses'] = _get_addresses(cfg, mconfig) return context
async def set_enodebd_iptables_rule(): """ Remove & Set iptable rules for exposing public IP for enobeb instead of private IP.. """ # Remove & Set iptable rules for exposing public ip # for enobeb instead of private cfg = load_service_config('enodebd') port, interface = cfg['tr069']['port'], cfg['tr069']['interface'] enodebd_public_ip = cfg['tr069']['public_ip'] # IPv4 only as iptables only works for IPv4. TODO: Investigate ip6tables? enodebd_ip = get_ip_from_if(interface, preference=IpPreference.IPV4_ONLY) # Incoming data from 192.88.99.142 -> enodebd address (eg 192.168.60.142) enodebd_netmask = get_if_ip_with_netmask( interface, preference=IpPreference.IPV4_ONLY, )[1] verify_config = does_iface_config_match_expected( enodebd_ip, enodebd_netmask, ) if not verify_config: logging.warning( 'The IP address of the %s interface is %s. The ' 'expected IP addresses are %s', interface, enodebd_ip, str(EXPECTED_IP4)) await check_and_apply_iptables_rules( port, enodebd_public_ip, enodebd_ip, )
def get_context(): """ Provide context to pass to Jinja2 for templating. """ context = {} cfg = load_service_config("dnsd") try: mconfig = load_service_mconfig("dnsd") except LoadConfigError as err: logging.warn("Error! Using default config because: %s", err) mconfig = DnsD() ip = get_ip_from_if_cidr(cfg['enodeb_interface']) dhcp_block_size = cfg['dhcp_block_size'] available_hosts = list(ipaddress.IPv4Interface(ip).network.hosts()) if dhcp_block_size < len(available_hosts): context['dhcp_range'] = { "lower": str(available_hosts[-dhcp_block_size]), "upper": str(available_hosts[-1]), } else: logging.fatal("Not enough available hosts to allocate a DHCP block of \ %d addresses." % (dhcp_block_size)) context['addresses'] = _get_addresses(cfg, mconfig) return context
def check_ipv6_iface_config(service, config_name, config_value): """Check if eth3 interface is configured""" service_config = load_service_config(service) if service_config.get(config_name) == config_value: print("IPV6_ENABLED\t%s -> %s" % (service, config_name)) return return_codes.IPV6_ENABLED return return_codes.IPV4_ENABLED
def ovs_reset_bridges(): mconfig = get_mconfig_manager().load_service_mconfig( 'pipelined', mconfigs_pb2.PipelineD(), ) service_config = load_service_config('pipelined') if service_config.get('enable_nat', mconfig.nat_enabled): non_nat_sgi_interface = "" sgi_management_iface_ip_addr = "" sgi_management_iface_gw = "" else: non_nat_sgi_interface = service_config['nat_iface'] sgi_management_iface_ip_addr = service_config.get( 'sgi_management_iface_ip_addr', mconfig.sgi_management_iface_ip_addr, ) sgi_management_iface_gw = service_config.get( 'sgi_management_iface_gw', mconfig.sgi_management_iface_gw, ) sgi_bridge_name = service_config.get('uplink_bridge', "uplink_br0") reset_br = "magma-bridge-reset.sh -n %s %s %s %s" % \ ( sgi_bridge_name, non_nat_sgi_interface, sgi_management_iface_ip_addr, sgi_management_iface_gw, ) print("ovs-restart: ", reset_br) subprocess.call(reset_br.split())
def debug(cls, _, __, ___): config = load_service_config('pipelined') qos_impl_type = QosImplType(config["qos"]["impl"]) qos_store = QosStore(cls.__name__) for k, v in qos_store.items(): _, imsi, ip_addr, rule_num, d = get_key(k) _, qid, ambr, leaf = get_data(v) print('imsi :', imsi) print('ip_addr :', ip_addr) print('rule_num :', rule_num) print('direction :', d) print('qos_handle:', qid) print('qos_handle_ambr:', ambr) print('qos_handle_ambr_leaf:', leaf) if qos_impl_type == QosImplType.OVS_METER: MeterManager.dump_meter_state(v) else: intf = 'nat_iface' if d == FlowMatch.UPLINK else 'enodeb_iface' print("Dev: ", config[intf]) TrafficClass.dump_class_state(config[intf], qid) if leaf and leaf != qid: print("Leaf:") TrafficClass.dump_class_state(config[intf], leaf) if ambr: print("AMBR (parent):") TrafficClass.dump_class_state(config[intf], ambr)
def debug(cls, _, __, ___): config = load_service_config('pipelined') qos_impl_type = QosImplType(config["qos"]["impl"]) qos_store = QosStore(cls.__name__, client=get_default_client()) for k, v in qos_store.items(): _, imsi, ip_addr, rule_num, d = get_key(k) _, qid, ambr, leaf = get_data(v) print('imsi :', imsi) print('ip_addr :', ip_addr) print('rule_num :', rule_num) print('direction :', d) print('qos_handle:', qid) print('qos_handle_ambr:', ambr) print('qos_handle_ambr_leaf:', leaf) if qos_impl_type == QosImplType.OVS_METER: MeterManager.dump_meter_state(v) else: dev = config[ 'nat_iface'] if d == FlowMatch.UPLINK else 'gtpu_sys_2152' print("Dev: ", dev) TrafficClass.dump_class_state(dev, qid) if leaf and leaf != qid: print("Leaf:") TrafficClass.dump_class_state(dev, leaf) if ambr: print("AMBR (parent):") TrafficClass.dump_class_state(dev, ambr) if qos_impl_type == QosImplType.LINUX_TC: dev = config['nat_iface'] print("Root stats for: ", dev) TrafficClass.dump_root_class_stats(dev) dev = 'gtpu_sys_2152' print("Root stats for: ", dev) TrafficClass.dump_root_class_stats(dev)
def main(): """ Top-level function for health service """ service = MagmaService('health', None) # Optionally pipe errors to Sentry sentry_init() # Service state wrapper obj service_state = ServiceStateWrapper() # Load service YML config state_recovery_config = service.config["state_recovery"] services_check = state_recovery_config["services_check"] polling_interval = int(state_recovery_config["interval_check_mins"]) * 60 restart_threshold = state_recovery_config["restart_threshold"] snapshots_dir = state_recovery_config["snapshots_dir"] redis_dump_src = load_service_config("redis").get("dir", "/var/opt/magma") state_recovery_job = StateRecoveryJob(service_state=service_state, polling_interval=polling_interval, services_check=services_check, restart_threshold=restart_threshold, redis_dump_src=redis_dump_src, snapshots_dir=snapshots_dir, service_loop=service.loop) state_recovery_job.start() # Run the service loop service.run() # Cleanup the service service.close()
async def download_update(target_image: str, git_hash: str) -> None: """ Download the images for the given tag and clones the github repo. """ await run_command("rm -rf {}".format(MAGMA_GITHUB_PATH), shell=True, check=True) await run_command("mkdir -p {}".format(MAGMA_GITHUB_PATH), shell=True, check=True) control_proxy_config = load_service_config('control_proxy') await run_command("cp {} /usr/local/share/ca-certificates/rootCA.crt". format(control_proxy_config['rootca_cert']), shell=True, check=True) await run_command("update-ca-certificates", shell=True, check=True) git_clone_cmd = "git -c http.proxy=https://{}:{} -C {} clone {}".format( control_proxy_config['bootstrap_address'], control_proxy_config['bootstrap_port'], MAGMA_GITHUB_PATH, MAGMA_GITHUB_URL) await run_command(git_clone_cmd, shell=True, check=True) git_checkout_cmd = "git -C {}/magma checkout {}".format( MAGMA_GITHUB_PATH, git_hash, ) await run_command(git_checkout_cmd, shell=True, check=True) docker_login_cmd = "docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD " \ "$DOCKER_REGISTRY" await run_command(docker_login_cmd, shell=True, check=True) docker_pull_cmd = "IMAGE_VERSION={} docker-compose --project-directory " \ "/var/opt/magma/docker -f " \ "/var/opt/magma/docker/docker-compose.yml pull -q".\ format(target_image) await run_command(docker_pull_cmd, shell=True, check=True)
def reload_config(self): """Reload the local config for the service""" try: self._config = load_service_config(self._name) self._setup_logging() except LoadConfigError as e: logging.warning(e)
def _display_flows(client, apps=None): pipelined_config = load_service_config('pipelined') bridge_name = pipelined_config['bridge_name'] response = client.GetAllTableAssignments(Void()) table_assignments = { table_assignment.app_name: Tables( main_table=table_assignment.main_table, type=None, scratch_tables=table_assignment.scratch_tables, ) for table_assignment in response.table_assignments } try: flows = BridgeTools.get_annotated_flows_for_bridge( bridge_name, table_assignments, apps, ) except subprocess.CalledProcessError as e: if e.returncode == errno.EPERM: print("Need to run as root to dump flows") return for flow in flows: print(flow)
def run(self) -> None: """ Create and start HTTP server """ svc_config = load_service_config("enodebd") app = web.Application() app.router.add_route( 'POST', "/{something}", self._post_and_put_handler, ) app.router.add_route( 'PUT', "/{something}", self._post_and_put_handler, ) handler = app.make_handler() create_server_func = self.loop.create_server( handler, host=get_ip_from_if(svc_config['tr069']['interface']), port=svc_config['tr069']['perf_mgmt_port'], ) self._periodic_check_rf_tx() self.loop.run_until_complete(create_server_func)
def check_stateless_service_config(service, config_name, config_value): service_config = load_service_config(service) if service_config.get(config_name) == config_value: print("STATELESS\t%s -> %s" % (service, config_name)) return return_codes.STATELESS print("STATEFUL\t%s -> %s" % (service, config_name)) return return_codes.STATEFUL
def _check_stateless_service_config(service, config_name, config_value): service_config = load_service_config(service) if service_config.get(config_name) == config_value: logging.info("STATELESS\t%s -> %s", service, config_name) return magmad_pb2.CheckStatelessResponse.STATELESS logging.info("STATEFUL\t%s -> %s", service, config_name) return magmad_pb2.CheckStatelessResponse.STATEFUL
def __init__(self, polling_interval: int, service_loop): super().__init__(interval=CHECKIN_INTERVAL, loop=service_loop) self._config = load_service_config("monitord") self._MTR_PORT = self._config["mtr_interface"] # Matching response time output to get latency self._polling_interval = max(polling_interval, DEFAULT_POLLING_INTERVAL) # TODO: Save to redis self._subscriber_state = defaultdict(ICMPMonitoringResponse) self._loop = service_loop
def get_streamer_status(_, args): if args.service not in STREAMER_KEYS_BY_SERVICE: print('No streamer config available for', args.service) return cfg = service_configs.load_service_config(args.service) if cfg is None: print('No config found!') for key in STREAMER_KEYS_BY_SERVICE[args.service]: enabled = cfg.get(key, False) print('%s: %s' % (key, enabled))
def create_parser(): """ Creates the argparse parser with all the arguments. """ pipelined_config = load_service_config('pipelined') bridge_name = pipelined_config['bridge_name'] ovs_gtp_port_number = pipelined_config['ovs_gtp_port_number'] parser = argparse.ArgumentParser( description='CLI wrapper around ovs-appctl ofproto/trace.\n' 'Note: this won\'t be able to trace packets in userspace', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-br', '--bridge_name', default=bridge_name, help='OVS bridge name(gtp_br0 or cwag_br0 or...)') subparsers = parser.add_subparsers(title='direction', dest='cmd') uplink = subparsers.add_parser('uplink', help='Trace packet going from UE') uplink.add_argument('-p', '--in_port', default=ovs_gtp_port_number, help='gtp port number') dlink = subparsers.add_parser('dlink', help='Trace packet coming from UPLINK') dlink.add_argument('-p', '--in_port', default='eth2', help='In port name') uplink.add_argument('tun_src', help='Tunnel src ip', default='') uplink.add_argument('-tun_dst', help='Tunnel dst ip', default='192.168.128.1') uplink.add_argument('-tun_id', help='Tunnel id(key)', default='') uplink.set_defaults(direction='OUT') dlink.set_defaults(direction='IN') for dir_p in [uplink, dlink]: dir_p.add_argument('ue_mac', help='UE mac address(unused in LTE)', default='0a:00:27:00:00:03') dir_p.add_argument('-ue_ip_addr', help='UE IP address', default='172.168.10.13') dir_p.add_argument('-uplink_mac', help='UPLINK mac address', default='00:27:12:00:33:00') dir_p.add_argument('--uplink_ip_addr', help='UPLINK IP address', default='104.28.26.94') proto_sp = dir_p.add_subparsers(title='protos', dest='cmd') tcp = proto_sp.add_parser('tcp', help='Specify tcp packet') tcp.add_argument('-ue_port', type=int, help='', default=3372) tcp.add_argument('-uplink_port', type=int, help='', default=80) tcp.add_argument('-seq', type=int, default=10001) tcp.add_argument('-ack', type=int, default=10002) tcp.add_argument('-tf', '--tcp_flags', help='Specify TCP flags', default='') icmp = proto_sp.add_parser('icmp', help='Specify icmp packet') icmp.add_argument('-t', '--type', type=int, help='ICMP type', default=0) icmp.add_argument('-c', '--code', type=int, help='ICMP code', default=0) tcp.set_defaults(func=_trace_tcp_pkt) icmp.set_defaults(func=_trace_icmp_pkt) return parser
def display_raw_flows(_unused, args): pipelined_config = load_service_config('pipelined') bridge_name = pipelined_config['bridge_name'] try: flows = BridgeTools.get_flows_for_bridge(bridge_name, args.table_num) except subprocess.CalledProcessError as e: if e.returncode == errno.EPERM: print("Need to run as root to dump flows") return for flow in flows: print(flow)
def get_proxy_config(): """ Returns the control proxy config. The config file is loaded only once and cached. """ if not ServiceRegistry._PROXY_CONFIG: try: ServiceRegistry._PROXY_CONFIG = load_service_config( 'control_proxy') except LoadConfigError as err: logging.error(err) ServiceRegistry._PROXY_CONFIG = {} return ServiceRegistry._PROXY_CONFIG
def get_registry(): """ Returns _REGISTRY which holds the contents from the config/service/service_registry.yml file. Its a static member and the .yml file is loaded only once. """ if not ServiceRegistry._REGISTRY: try: ServiceRegistry._REGISTRY = load_service_config( "service_registry", ) except LoadConfigError as err: logging.error(err) ServiceRegistry._REGISTRY = {"services": {}} return ServiceRegistry._REGISTRY
def tr069_server(state_machine_manager: StateMachineManager) -> None: """ TR-069 server Inputs: - acs_to_cpe_queue = instance of Queue containing messages from parent process/thread to be sent to CPE - cpe_to_acs_queue = instance of Queue containing messages from CPE to be sent to parent process/thread """ config = load_service_config("enodebd") AutoConfigServer.set_state_machine_manager(state_machine_manager) app = Tr069Application( [AutoConfigServer], CWMP_NS, in_protocol=Tr069Soap11(validator='soft'), out_protocol=Tr069Soap11(), ) wsgi_app = WsgiApplication(app) try: ip_address = get_ip_from_if(config['tr069']['interface']) except (ValueError, KeyError) as e: # Interrupt main thread since process should not continue without TR-069 _thread.interrupt_main() raise e socket.setdefaulttimeout(SOCKET_TIMEOUT) logger.info( 'Starting TR-069 server on %s:%s', ip_address, config['tr069']['port'], ) server = make_server( ip_address, config['tr069']['port'], wsgi_app, WSGIServer, tr069_WSGIRequestHandler, ) # Note: use single-thread server, to avoid state contention try: server.serve_forever() finally: # Log error and interrupt main thread, to ensure that entire process # is restarted if this thread exits logger.error('Hit error in TR-069 thread. Interrupting main thread.') _thread.interrupt_main()
def debug(cls, _): config = load_service_config('pipelined') qos_impl_type = QosImplType(config["qos"]["impl"]) qos_store = QosStore(cls.__name__) for k, v in qos_store.items(): _, imsi, rule_num, d = get_key(k) print('imsi :', imsi) print('rule_num :', rule_num) print('direction :', d) print('qos_handle:', v) if qos_impl_type == QosImplType.OVS_METER: MeterManager.dump_meter_state(v) else: intf = 'nat_iface' if d == FlowMatch.UPLINK else 'enodeb_iface' TrafficClass.dump_class_state(config[intf], v)
def run(self): """ Create and start HTTP server """ svc_config = load_service_config("enodebd") app = web.Application() app.router.add_route('POST', "/{something}", self.post_handler) loop = asyncio.get_event_loop() handler = app.make_handler() create_server_func = loop.create_server( handler, host=get_ip_from_if(svc_config['tr069']['interface']), port=svc_config['tr069']['perf_mgmt_port']) loop.run_until_complete(create_server_func)
def set_enodebd_iptables_rule(): """ Remove & Set iptable rules for exposing public IP for enobeb instead of private IP.. """ # Remove & Set iptable rules for exposing public ip # for enobeb instead of private cfg = load_service_config('enodebd') port, interface = cfg['tr069']['port'], cfg['tr069']['interface'] enodebd_public_ip = cfg['tr069']['public_ip'] # IPv4 only as iptables only works for IPv4. TODO: Investigate ip6tables? enodebd_ip = get_ip_from_if(interface, preference=IpPreference.IPV4_ONLY) # Incoming data from 192.88.99.142 -> enodebd address (eg 192.168.60.142) yield from run(get_iptables_rule( port, enodebd_public_ip, enodebd_ip, add=False)) yield from run(get_iptables_rule( port, enodebd_public_ip, enodebd_ip, add=True))
def main(): """ This module is used for manual testing of the TR-069 server """ config = load_service_config("enodebd") app = Tr069Application([AutoConfigServer], CWMP_NS, in_protocol=Tr069Soap11(validator="soft"), out_protocol=Tr069Soap11()) ip_address = get_ip_from_if(config['tr069']['interface']) client = Tr069HttpClient( "http://%s:%s" % (ip_address, config["tr069"]["port"]), app) client.set_options(out_header=ID("123", mustUnderstand="1")) rpc_methods = client.service.get_rpc_methods() for rpc_method in rpc_methods: print("Method: %s" % rpc_method) inform_req = client.factory.create("Inform") inform_req.DeviceId = client.factory.create("DeviceIdStruct") inform_req.DeviceId.Manufacturer = "Magma" inform_req.DeviceId.OUI = "ABCDEF" inform_req.DeviceId.ProductClass = "TopClass" inform_req.DeviceId.SerialNumber = "123456789" inform_req.Event = None inform_req.MaxEnvelopes = 1 inform_req.CurrentTime = None inform_req.RetryCount = 4 inform_req.ParameterList = None client.set_options(out_header=ID("456", mustUnderstand="1")) client.service.Inform(inform_req) dummy = DummyInput() dummy.Field1 = 5 rsp = client.service.EmptyHttp(dummy) print("EmptyHttp response = ", rsp) paramNames = client.factory.create("GetParameterNamesResponse") paramNames.ParameterList = client.factory.create("ParameterInfoList") paramNames.ParameterList.ParameterInfoStruct =\ [client.factory.create("ParameterInfoStruct")] paramNames.ParameterList.ParameterInfoStruct[0].Name = "Parameter1" paramNames.ParameterList.ParameterInfoStruct[0].Writable = True rsp = client.service.GetParameterNamesResponse(paramNames) print("GetParameterNamesResponse response = ", rsp)
def generate_template_config(service, template, out_dirname, context): """ Generate the config from the jinja template. Args: service (str): Name of the magma service. Used for looking up the config and mconfig template (str): Name of the input template, which is also used for choosing the output filename out_dirname (str): Path of the output file context (map): Context to use for Jinja (the .yml config and mconfig will be added into this context) """ # Get the template and the output filenames template_filename = _get_template_filename(template) out_filename = _get_template_out_filename(template, out_dirname) logging.info( "Generating config file: [%s] using template: [%s]" % ( out_filename, template_filename, ), ) template_context = {} # Generate the content to use from the service yml config and mconfig. try: template_context.update(load_service_config(service)) except LoadConfigError as err: logging.warning(err) template_context.update(context) try: mconfig = load_service_mconfig_as_json(service) template_context.update(mconfig) except LoadConfigError as err: logging.warning(err) # Export snowflake to template. # TODO: export a hardware-derived ID that can be used by a field tech # to easily identify a specific device. template_context.setdefault("snowflake", make_snowflake()) # Create the config file based on the template template_str = open(template_filename, 'r').read() output = Template(template_str).render(template_context) os.makedirs(out_dirname, exist_ok=True) write_to_file_atomically(out_filename, output)
def __init__(self, service, bootstrap_success_cb): super().__init__( self.PERIODIC_BOOTSTRAP_CHECK_INTERVAL.total_seconds(), service.loop) control_proxy_config = load_service_config('control_proxy') self._challenge_key_file \ = service.config['bootstrap_config']['challenge_key'] self._hw_id = snowflake.snowflake() self._gateway_key_file = control_proxy_config['gateway_key'] self._gateway_cert_file = control_proxy_config['gateway_cert'] self._gateway_key = None self._state = BootstrapState.INITIAL self._bootstrap_success_cb = bootstrap_success_cb # give some margin on watchdog check interval self.set_timeout(self._interval * 1.1)
def __init__(self, name, loop=None): self._name = name self._port = 0 self._get_status_callback = None self._reload_config_callback = None # Init logging before doing anything logging.basicConfig( level=logging.INFO, format='[%(asctime)s %(levelname)s %(name)s] %(message)s') # Set gRPC polling strategy self._set_grpc_poll_strategy() # Load the managed config if present self._mconfig = None self._mconfig_manager = get_mconfig_manager() self.reload_mconfig() self._state = ServiceInfo.STARTING self._health = ServiceInfo.APP_UNHEALTHY if loop is None: loop = asyncio.get_event_loop() self._loop = loop self._start_time = int(time.time()) self._register_signal_handlers() # Load the service config if present self._config = None try: self._config = load_service_config(name) except LoadConfigError as e: logging.warning(e) self._setup_logging() self._version = '0.0.0' # Load the service version if available try: self._version = pkg_resources.get_distribution('orc8r').version except pkg_resources.ResolutionError as e: logging.info(e) self._server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) add_Service303Servicer_to_server(self, self._server)