def get_client(self): """ get_client returns a grpc client of the specified service in the cloud. it will return a recycled client until the client fails or the number of recycling reaches the max_client_use. """ if self._client is None or \ self._num_client_use > self._max_client_reuse: chan = ServiceRegistry.get_rpc_channel( self._service_name, ServiceRegistry.CLOUD, ) self._client = self._service_stub(chan) self._num_client_use = 0 self._num_client_use += 1 return self._client
def get_magma_services_summary(self): """ Get health for all the running services """ services_health_summary = [] # DBus objects: https://www.freedesktop.org/wiki/Software/systemd/dbus/ chan = ServiceRegistry.get_rpc_channel('magmad', ServiceRegistry.LOCAL) client = MagmadStub(chan) configs = client.GetConfigs(common_pb2.Void()) service_names = [str(name) for name in configs.configs_by_key] services_errors = self.get_error_summary(service_names=service_names) for service_name in service_names: unit = Unit( 'magma@{}.service'.format(service_name), _autoload=True, ) active_state = ActiveState.dbus2state[unit.Unit.ActiveState] sub_state = str(unit.Unit.SubState, 'utf-8') if active_state == ActiveState.ACTIVE: pid = unit.Service.MainPID process = subprocess.Popen( 'ps -o etime= -p {}'.format(pid).split(), stdout=subprocess.PIPE, ) time_running, error = process.communicate() if error: raise ValueError( 'Cannot get time running for the service ' '{} `ps -o etime= -p {}`' .format(service_name, pid), ) else: time_running = b'00' services_health_summary.append( ServiceHealth( service_name=service_name, active_state=active_state, sub_state=sub_state, time_running=str(time_running, 'utf-8').strip(), errors=services_errors[service_name], ), ) return services_health_summary
def _checkin(self, service_statusmeta): """ if previous checkin is successful, create a new channel (to make sure the channel does't become stale). Otherwise, keep the existing channel. """ if self._checkin_client is None: chan = ServiceRegistry.get_rpc_channel( 'checkind', ServiceRegistry.CLOUD) self._checkin_client = CheckindStub(chan) mconfig = self._service.mconfig cpu = psutil.cpu_times() mem = psutil.virtual_memory() try: gw_ip = get_ip_from_if('tun0') # look for tun0 interface except ValueError: gw_ip = 'N/A' request = CheckinRequest( gateway_id=snowflake.snowflake(), magma_pkg_version=self._service.version, system_status=SystemStatus( cpu_user=int(cpu.user * 1000), # convert second to millisecond cpu_system=int(cpu.system * 1000), cpu_idle=int(cpu.idle * 1000), mem_total=mem.total, mem_available=mem.available, mem_used=mem.used, mem_free=mem.free, uptime_secs=int(time.time() - self._boot_time), ), vpn_ip=gw_ip, kernel_version=self._kernel_version, kernel_versions_installed=self._kernel_versions_installed, ) for statusmeta in service_statusmeta.values(): request.status.meta.update(statusmeta) future = self._checkin_client.Checkin.future( request, mconfig.checkin_timeout, ) future.add_done_callback( lambda f: self._loop.call_soon_threadsafe(self._checkin_done, f), )
def set_mobilityd_gw_info(ip: IPAddress, mac: str, vlan: str): """ Make RPC call to 'SetGatewayInfo' method of local mobilityD service """ try: chan = ServiceRegistry.get_rpc_channel(SERVICE_NAME, ServiceRegistry.LOCAL) except ValueError: logging.error('Cant get RPC channel to %s', SERVICE_NAME) return client = MobilityServiceStub(chan) try: gwinfo = GWInfo(ip=ip, mac=mac, vlan=vlan) client.SetGatewayInfo(gwinfo) except grpc.RpcError as err: logging.error("SetGatewayInfo error[%s] %s", err.code(), err.details())
def get_service303_client(service_name: str, location: str) \ -> Optional[Service303Stub]: """ get_service303_client returns a grpc client attached to the given service name and location. Example Use: client = get_service303_client("state", ServiceRegistry.LOCAL) """ try: chan = ServiceRegistry.get_rpc_channel( service_name, location, ) return Service303Stub(chan) except ValueError: # Service can't be contacted logging.error('Failed to get RPC channel to %s', service_name) return None
def get_all_enb_connected() -> Optional[List[int]]: """ Make RPC call to 'GetEnbConnected' method of s1ap service """ try: chan = ServiceRegistry.get_rpc_channel(S1AP_SERVICE_NAME, ServiceRegistry.LOCAL) except ValueError: logger.error('Cant get RPC channel to %s', S1AP_SERVICE_NAME) return client = S1apServiceStub(chan) try: res = client.GetEnbConnected(Void(), DEFAULT_GRPC_TIMEOUT) return res.enb_ids except grpc.RpcError as err: logger.warning("GetEnbConnected error: [%s] %s", err.code(), err.details()) return []
def get_mobilityd_gw_info() -> List[GWInfo]: """ Make RPC call to 'GetGatewayInfo' method of local mobilityD service """ try: chan = ServiceRegistry.get_rpc_channel(SERVICE_NAME, ServiceRegistry.LOCAL) except ValueError: logging.error('Cant get RPC channel to %s', SERVICE_NAME) return GWInfo() client = MobilityServiceStub(chan) try: return client.ListGatewayInfo(Void()).gw_list except grpc.RpcError as err: logging.error("ListGatewayInfo error[%s] %s", err.code(), err.details()) return []
async def _get_subscribers(self) -> List[IPAddress]: """ Sends gRPC call to mobilityd to get all subscribers table. Returns: List of [Subscriber ID => IP address, APN] entries """ try: mobilityd_chan = ServiceRegistry.get_rpc_channel( 'mobilityd', ServiceRegistry.LOCAL) mobilityd_stub = MobilityServiceStub(mobilityd_chan) response = await grpc_async_wrapper( mobilityd_stub.GetSubscriberIPTable.future( Void(), TIMEOUT_SECS), self._loop) return response.entries except grpc.RpcError as err: logging.error("GetSubscribers Error for %s! %s", err.code(), err.details()) return []
def get_allocated_ips(): chan = ServiceRegistry.get_rpc_channel('mobilityd', ServiceRegistry.LOCAL) client = MobilityServiceStub(chan) res = [] list_blocks_resp = client.ListAddedIPv4Blocks(Void()) for block_msg in list_blocks_resp.ip_block_list: list_ips_resp = client.ListAllocatedIPs(block_msg) for ip_msg in list_ips_resp.ip_list: if ip_msg.version == IPAddress.IPV4: ip = ipaddress.IPv4Address(ip_msg.address) elif ip_msg.address == IPAddress.IPV6: ip = ipaddress.IPv6Address(ip_msg.address) else: continue res.append(ip) return res
def main(): """ main() for subscriberdb """ service = MagmaService('brokerd', mconfigs_pb2.BrokerD()) # Add all servicers to the server chan = ServiceRegistry.get_rpc_channel('subscriberdb', ServiceRegistry.LOCAL) s6a_proxy_stub = S6aProxyStub(chan) brokerd_servicer = BrokerdRpcServicer(s6a_proxy_stub) brokerd_servicer.add_to_server(service.rpc_server) logging.info('brokerd is running!') # Run the service loop service.run() # Cleanup the service service.close()
def sync(self): """ Synchronizes sample queue to cloud and reschedules sync loop """ if self._samples: chan = ServiceRegistry.get_rpc_channel('metricsd', ServiceRegistry.CLOUD) client = MetricsControllerStub(chan) samples = self._retry_queue + self._samples metrics_container = MetricsContainer( gatewayId=snowflake.snowflake(), family=samples) future = client.Collect.future(metrics_container, self.grpc_timeout) future.add_done_callback( lambda future: self._loop.call_soon_threadsafe( self.sync_done, samples, future)) self._retry_queue.clear() self._samples.clear() self._loop.call_later(self.sync_interval, self.sync)
def _package_and_send_metrics(self, metrics: [metrics_pb2.MetricFamily], target: ScrapeTarget) -> None: """ Send parsed and protobuf-converted metrics to cloud. """ metrics_container = MetricsContainer( gatewayId=snowflake.snowflake(), family=metrics, ) chan = ServiceRegistry.get_rpc_channel('metricsd', ServiceRegistry.CLOUD, grpc_options=self._grpc_options) client = MetricsControllerStub(chan) future = client.Collect.future(metrics_container, self.grpc_timeout) future.add_done_callback( lambda future: self._loop.call_soon_threadsafe( self.scrape_done, future, target))
def gateway_health_status(self): config = load_service_mconfig_as_json('mme') # eNB status for #eNBs connected chan = ServiceRegistry.get_rpc_channel('enodebd', ServiceRegistry.LOCAL) client = EnodebdStub(chan) status = client.GetStatus(Void()) mme_log_path = '/var/log/mme.log' health_summary = AGWHealthSummary( relay_enabled=config['relayEnabled'], nb_enbs_connected=status.meta['n_enodeb_connected'], allocated_ips=self.get_allocated_ips(), subscriber_table=self.get_subscriber_table(), registration_success_rate=self.get_registration_success_rate( mme_log_path), ) return health_summary
def detach_deleted_subscribers(self, old_sub_ids, new_sub_ids): """ Compares current subscriber ids and new subscriber ids list just streamed from the cloud to figure out the deleted subscribers. Then send grpc DeleteSubscriber request to mme to detach all the deleted subscribers. :param old_sub_ids: a list of old subscriber ids in the store. :param new_sub_ids: a list of new active subscriber ids just streamed from the cloud :return: n/a """ # THIS IS A HACK UNTIL WE FIX THIS ON CLOUD # We accept IMSIs with or without 'IMSI' prepended on cloud, but we # always store IMSIs on local subscriberdb with IMSI prepended. If the # cloud streams down subscriber IDs without 'IMSI' prepended, # subscriberdb will try to delete all of the subscribers from MME every # time it streams from cloud because the set membership will fail # when comparing '12345' to 'IMSI12345'. new_sub_ids = set( map( lambda s: 'IMSI' + s if not s.startswith('IMSI') else s, new_sub_ids, ), ) deleted_sub_ids = [ sub_id for sub_id in old_sub_ids if sub_id not in set(new_sub_ids) ] if len(deleted_sub_ids) == 0: return # send detach request to mme for all deleted subscribers. chan = ServiceRegistry.get_rpc_channel('s6a_service', ServiceRegistry.LOCAL) client = S6aServiceStub(chan) req = DeleteSubscriberRequest() # mme expects a list of IMSIs without "IMSI" prefix imsis_to_delete_without_prefix = [sub[4:] for sub in deleted_sub_ids] req.imsi_list.extend(imsis_to_delete_without_prefix) future = client.DeleteSubscriber.future(req) future.add_done_callback( lambda future: self._loop.call_soon_threadsafe( self.detach_deleted_subscribers_done, future))
def run(self): """ This is executed when the thread is started. It gets a connection to the cloud dispatcher, and calls its bidirectional streaming rpc EstablishSyncRPCStream(). process_streams should never return, and if it did, exception will be logged, and new connection to dispatcher will be attempted after RETRY_DELAY_SECS seconds. """ while True: try: start_time = time.time() chan = ServiceRegistry.get_rpc_channel( 'dispatcher', ServiceRegistry.CLOUD, ) client = SyncRPCServiceStub(chan) self._set_connect_time() self.process_streams(client) except grpc.RpcError as err: if is_grpc_error_retryable(err): logging.warning( "[SyncRPC] Transient gRPC error, retrying: %s", err.details(), ) self._retry_connect_sleep() continue else: logging.error( "[SyncRPC] gRPC error: %s, reconnecting to cloud.", err.details(), extra=EXCLUDE_FROM_ERROR_MONITORING, ) self._cleanup_and_reconnect() except Exception as exp: # pylint: disable=broad-except conn_time = time.time() - start_time logging.error( "[SyncRPC] Error after %ds: %s", conn_time, exp, extra=EXCLUDE_FROM_ERROR_MONITORING, ) self._cleanup_and_reconnect()
async def test_checkin(proxy_cloud_connections=True): """Send checkin using either proxy or direct to cloud connection""" chan = ServiceRegistry.get_rpc_channel( 'state', ServiceRegistry.CLOUD, proxy_cloud_connections=proxy_cloud_connections, ) client = StateServiceStub(chan) # Construct a simple state to send for test value = json.dumps({"datetime": datetime.datetime.now()}, default=str) states = [ State(type="string_map", deviceID=snowflake.snowflake(), value=value.encode('utf-8')), ] request = ReportStatesRequest(states=states) timeout = 1000 await grpc_async_wrapper(client.ReportStates.future(request, timeout))
def collect(self, service_name): """ Calls into Service303 to get service metrics samples and reschedule collection. """ chan = ServiceRegistry.get_rpc_channel( service_name, ServiceRegistry.LOCAL, ) client = Service303Stub(chan) future = client.GetMetrics.future(Void(), self.grpc_timeout) future.add_done_callback( lambda future: self._loop.call_soon_threadsafe( self.collect_done, service_name, future, ), ) self._loop.call_later( self.collect_interval, self.collect, service_name, )
def get_all_records(retries: int = 3, sleep_time: float = 0.1) -> [dict]: """ Make RPC call to 'GetAllDirectoryRecords' method of local directoryD service """ try: chan = ServiceRegistry.get_rpc_channel(DIRECTORYD_SERVICE_NAME, ServiceRegistry.LOCAL) except ValueError: logging.error('Cant get RPC channel to %s', DIRECTORYD_SERVICE_NAME) return client = GatewayDirectoryServiceStub(chan) for _ in range(0, retries): try: res = client.GetAllDirectoryRecords(Void(), DEFAULT_GRPC_TIMEOUT) if res.records is not None: return res.records hub.sleep(sleep_time) except grpc.RpcError as err: logging.error("GetAllDirectoryRecords error! [%s] %s", err.code(), err.details()) return []
def log_event(event: Event) -> None: """ Make RPC call to 'LogEvent' method of local eventD service """ try: chan = ServiceRegistry.get_rpc_channel(EVENTD_SERVICE_NAME, ServiceRegistry.LOCAL) except ValueError: logging.error("Cant get RPC channel to %s", EVENTD_SERVICE_NAME) return client = EventServiceStub(chan) try: # Location will be filled in by directory service client.LogEvent(event, DEFAULT_GRPC_TIMEOUT) except grpc.RpcError as err: logging.error( "LogEvent error for event: %s, [%s] %s", MessageToDict(event), err.code(), err.details(), )
def main(): """ main() for ctraced """ service = MagmaService('ctraced', CtraceD()) # Optionally pipe errors to Sentry sentry_init() orc8r_chan = ServiceRegistry.get_rpc_channel('ctraced', ServiceRegistry.CLOUD) ctraced_stub = CallTraceControllerStub(orc8r_chan) trace_manager = TraceManager(service.config, ctraced_stub) ctraced_servicer = CtraceDRpcServicer(trace_manager) ctraced_servicer.add_to_server(service.rpc_server) # Run the service loop service.run() # Cleanup the service service.close()
def _load_subs(num_subs: int) -> List[DirectoryRecord]: """Load directory records""" client = GatewayDirectoryServiceStub( ServiceRegistry.get_rpc_channel( DIRECTORYD_SERVICE_NAME, ServiceRegistry.LOCAL, ), ) sids = [] for i in range(num_subs): mac_addr = (str(i) * 2 + ":") * 5 + (str(i) * 2) ipv4_addr = str(i) * 3 + "." + str(i) * 3 + "." + str( i) * 3 + "." + str(i) * 3 fields = {"mac-addr": mac_addr, "ipv4_addr": ipv4_addr} sid = UpdateRecordRequest( fields=fields, id=str(i).zfill(15), location=str(i).zfill(15), ) client.UpdateRecord(sid) sids.append(sid) return sids
def test_service_run(self, mock_get_proxy_config): """ Test if the service starts and stops gracefully. """ self.assertEqual(self._service.state, ServiceInfo.STARTING) mock_get_proxy_config.return_value = { 'cloud_address': '127.0.0.1', 'proxy_cloud_connections': True, } # Start the service and pause the loop self._service.loop.stop() self._service.run() asyncio.set_event_loop(self._service.loop) self._service.log_counter._periodic_task.cancel() self.assertEqual(self._service.state, ServiceInfo.ALIVE) # Create a rpc stub and query the Service303 interface ServiceRegistry.add_service('test', '0.0.0.0', self._service.port) channel = ServiceRegistry.get_rpc_channel( 'test', ServiceRegistry.LOCAL, ) self._stub = Service303Stub(channel) info = ServiceInfo( name='test', version='0.0.0', state=ServiceInfo.ALIVE, health=ServiceInfo.APP_HEALTHY, start_time_secs=12345, ) self.assertEqual(self._stub.GetServiceInfo(Void()), info) # Stop the service self._stub.StopService(Void()) self._service.loop.run_forever() self.assertEqual(self._service.state, ServiceInfo.STOPPED)
async def _get_service_info(self): """ Make RPC calls to 'GetServiceInfo' functions of other services, to get current status. """ for service in list(self._service_info): # Check whether service provides service303 interface if service in self._config['non_service303_services']: continue try: chan = ServiceRegistry.get_rpc_channel( service, ServiceRegistry.LOCAL, ) except ValueError: # Service can't be contacted logging.error('Cant get RPC channel to %s', service) continue client = Service303Stub(chan) try: future = client.GetServiceInfo.future( Void(), self.GET_STATUS_TIMEOUT, ) info = await grpc_async_wrapper(future, self._loop) self._service_info[service].update( info.start_time_secs, info.status, ) self._service_info[service].continuous_timeouts = 0 except grpc.RpcError as err: logging.error( "GetServiceInfo Error for %s! [%s] %s", service, err.code(), err.details(), extra=EXCLUDE_FROM_ERROR_MONITORING if indicates_connection_error(err) else None, ) self._service_info[service].continuous_timeouts += 1
def activate_he_urls_for_ue( ip: IPAddress, rule_id: str, urls: List[str], imsi: str, msisdn: str, ) -> bool: """ Make RPC call to 'Envoy Controller' to add target URLs to envoy datapath. """ try: chan = ServiceRegistry.get_rpc_channel( SERVICE_NAME, ServiceRegistry.LOCAL, ) except grpc.RpcError: logging.error('Cant get RPC channel to %s', SERVICE_NAME) return False client = EnvoyControllerStub(chan) try: headers = [Header(name=IMSI_HDR, value=imsi)] if msisdn: headers.append(Header(name=MSISDN_HDR, value=msisdn)) he_info = AddUEHeaderEnrichmentRequest( ue_ip=ip, rule_id=rule_id, websites=urls, headers=headers, ) ret = client.AddUEHeaderEnrichment(he_info, timeout=TIMEOUT_SEC) return ret.result == AddUEHeaderEnrichmentResult.SUCCESS except grpc.RpcError as err: logging.error( "Activate HE proxy error[%s] %s", err.code(), err.details(), ) return False
def sync(self, service_name): """ Synchronizes sample queue for specific service to cloud and reschedules sync loop """ if service_name in self._samples_for_service and \ self._samples_for_service[service_name]: chan = ServiceRegistry.get_rpc_channel( 'metricsd', ServiceRegistry.CLOUD, grpc_options=self._grpc_options, ) client = MetricsControllerStub(chan) if self.post_processing_fn: # If services wants to, let it run a postprocessing function # If we throw an exception here, we'll have no idea whether # something was postprocessed or not, so I guess try and make it # idempotent? #m sevchicken self.post_processing_fn( self._samples_for_service[service_name], ) samples = self._samples_for_service[service_name] sample_chunks = self._chunk_samples(samples) for idx, chunk in enumerate(sample_chunks): metrics_container = MetricsContainer( gatewayId=snowflake.snowflake(), family=chunk, ) future = client.Collect.future( metrics_container, self.grpc_timeout, ) future.add_done_callback( self._make_sync_done_func( service_name, idx, ), ) self._samples_for_service[service_name].clear() self._loop.call_later(self.sync_interval, self.sync, service_name)
async def get_ping_targets(self, service_loop) -> PingedTargets: """ Sends gRPC call to mobilityd to get all subscribers table. Returns: List of [Subscriber ID => IP address, APN] entries """ try: mobilityd_chan = ServiceRegistry.get_rpc_channel( 'mobilityd', ServiceRegistry.LOCAL) mobilityd_stub = MobilityServiceStub(mobilityd_chan) response = await grpc_async_wrapper( mobilityd_stub.GetSubscriberIPTable.future(Void(), 10), service_loop) for sub in response.entries: ip = _get_addr_from_subscribers(sub.ip) self.ping_addresses.append(ip) self.ping_targets[sub.sid.id] = ip except grpc.RpcError as err: logging.error("GetSubscribers Error for %s! %s", err.code(), err.details()) return PingedTargets(self.ping_targets, self.ping_addresses)
def update_record(imsi: str, ip_addr: str) -> None: """ Make RPC call to 'UpdateRecord' method of local directoryD service """ try: chan = ServiceRegistry.get_rpc_channel(DIRECTORYD_SERVICE_NAME, ServiceRegistry.LOCAL) except ValueError: logging.error('Cant get RPC channel to %s', DIRECTORYD_SERVICE_NAME) return client = GatewayDirectoryServiceStub(chan) if not imsi.startswith("IMSI"): imsi = "IMSI" + imsi try: # Location will be filled in by directory service req = UpdateRecordRequest(id=imsi, location="hwid") req.fields[IPV4_ADDR_KEY] = ip_addr client.UpdateRecord(req, DEFAULT_GRPC_TIMEOUT) except grpc.RpcError as err: logging.error( "UpdateRecordRequest error for id: %s, ipv4_addr: %s! [%s] %s", imsi, ip_addr, err.code(), err.details())
def run(self): """ This is executed when the thread is started. It gets a connection to the cloud dispatcher, and calls its bidirectional streaming rpc EstablishSyncRPCStream(). process_streams should never return, and if it did, exception will be logged, and new connection to dispatcher will be attempted after RETRY_DELAY_SECS seconds. """ while True: try: start_time = time.time() chan = ServiceRegistry.get_rpc_channel('dispatcher', ServiceRegistry.CLOUD) client = SyncRPCServiceStub(chan) self._set_connect_time() self.process_streams(client) except Exception as exp: # pylint: disable=broad-except conn_time = time.time() - start_time logging.error("[SyncRPC] Error after %ds: %s", conn_time, exp) # If the connection is terminated, wait for a period of time # before connecting back to the cloud. self._retry_connect_sleep()
def set_he_urls_for_ue(ip: str, urls: List[str], imsi: str, msisdn: str): """ Make RPC call to 'SetGatewayInfo' method of local mobilityD service """ try: chan = ServiceRegistry.get_rpc_channel(SERVICE_NAME, ServiceRegistry.LOCAL) except grpc.RpcError: logging.error('Cant get RPC channel to %s', SERVICE_NAME) return client = EnvoyControllerStub(chan) try: h1 = {IMSI_HDR: imsi} h2 = {MSISDN_HDR: msisdn} he_info = AddUEHeaderEnrichmentRequest(ue_ip=ip, websites=urls, headers=[h1, h2]) client.AddUEHeaderEnrichment(he_info) except grpc.RpcError as err: logging.error("SetGatewayInfo error[%s] %s", err.code(), err.details())
def get_cbsd_state(request: CBSDRequest) -> CBSDStateResult: """ Make RPC call to 'GetCBSDState' method of dp service """ try: chan = ServiceRegistry.get_rpc_channel( DP_SERVICE_NAME, ServiceRegistry.CLOUD, ) except ValueError: logger.error('Cant get RPC channel to %s', DP_SERVICE_NAME) return CBSDStateResult(radio_enabled=False) client = DPServiceStub(chan) try: res = client.GetCBSDState(request, DEFAULT_GRPC_TIMEOUT) except grpc.RpcError as err: logger.warning( "GetCBSDState error: [%s] %s", err.code(), err.details(), ) return CBSDStateResult(radio_enabled=False) return res