def upgrade_endaga(channel): """Upgrades the endaga metapackage.""" # Validate. if channel not in ('stable', 'beta'): logger.error('cannot upgrade to the "%s" channel' % channel) return logger.notice('upgrading the endaga metapackage with channel %s' % channel) # Update packages. response = delegator.run('sudo apt-get update') if response.return_code != 0: message = 'Error while running "apt-get update": %s' % response.out logger.error(message) # Try a dry-run of the upgrade. command = ('sudo apt-get install --assume-yes --dry-run' ' --only-upgrade -t %s endaga' % channel) response = delegator.run(command) if response.return_code != 0: message = ('Error while dry running the endaga upgrade: %s' % response.out) logger.error(message) return # Upgrade just the metapackage. command = ('sudo apt-get install --assume-yes' ' --only-upgrade -t %s endaga' % channel) response = delegator.run(command) if response.return_code != 0: message = 'Error while upgrading endaga: %s' % response.out logger.error(message)
def req_log(self, data): PERMITTED_LOGS = ['endaga', 'syslog'] required_fields = ["start", "end", "log_name"] if not all([_ in data for _ in required_fields]): return web.BadRequest() # By default there are no window start or end window_start = None window_end = None try: window_start = dateutil.parser.parse(data['start']) except ValueError: pass try: window_end = dateutil.parser.parse(data['end']) except ValueError: pass if data['log_name'] not in PERMITTED_LOGS: return web.BadRequest() logger.notice( "Log %s requested by dashboard from %s to %s" % (data['log_name'], window_start or '-', window_end or '-')) t = threading.Thread(target=self.log_worker, args=(data['msgid'], window_start, window_end, data['log_name'])) t.start() return web.Accepted()
def __init__(self): self._conf = ConfigDB() # initialise logging level from DB (if set - otherwise 'warning') # NB - this is the ONLY time changes to the log level are actually # passed to the logging framework log_level = self._conf.get("logger.global.log_level", "warning") logger.DefaultLogger.update_handler(level=log_level) logger.notice("EndagaD started")
def process_update(self, net_subs): """ Processes the subscriber list. Format is: { IMSI1: {'number': [<numbers>,...], 'balance': {<PN counter>}}, IMSI2: ... } This updates the BTS with all subscribers instructed by the cloud; any subscribers that are not reported by the cloud will be removed from this BTS. """ # dict where keys are imsis and values are sub info bts_imsis = self.get_subscriber_imsis() net_imsis = set(net_subs.keys()) subs_to_add = net_imsis.difference(bts_imsis) subs_to_delete = bts_imsis.difference(net_imsis) subs_to_update = bts_imsis.intersection(net_imsis) for imsi in subs_to_delete: self.delete_subscriber(imsi) # TODO(shasan) does not add new numbers for imsi in subs_to_update: sub = net_subs[imsi] try: bal = crdt.PNCounter.from_state(sub['balance']) self.update_balance(imsi, bal) except SubscriberNotFound as e: logger.warning( "Balance sync fail! IMSI: %s is not found Error: %s" % (imsi, e)) except ValueError as e: logger.error("Balance sync fail! IMSI: %s, %s Error: %s" % (imsi, sub['balance'], e)) subs_to_add.add(imsi) # try to add it (again) for imsi in subs_to_add: sub = net_subs[imsi] numbers = sub['numbers'] if not numbers: logger.notice("IMSI with no numbers? %s" % imsi) continue self.create_subscriber(imsi, numbers[0]) for n in numbers[1:]: self.add_number(imsi, n) try: bal = crdt.PNCounter.from_state(sub['balance']) self.update_balance(imsi, bal) except (SubscriberNotFound, ValueError) as e: logger.error("Balance sync fail! IMSI: %s, %s Error: %s" % (imsi, sub['balance'], e))
def _reset_bts_config(self): logger.notice("Performing set_factory") try: if bts.set_factory_config(): logger.notice("Restarting BTS") bts.restart() Service.SystemService("freeswitch").restart() except BSSError as e: logger.error("bts is probably down: %s" % e) except Exception as e: # OSError, IOError or whatever envoy will raise logger.critical("something unexpected happened: %s" % e)
def update_vpn(): """ If the BTS is registered, try to start the VPN. If the BTS is not registered, skip. If the BTS is unregistered (on the dashboard), no services are available. Regardless of whether the VPN is up or down, all services should be started (this will enable disconnected operation). However, when the VPN comes online, we need to restart FS to make sure that we're bound to the VPN IP so outgoing calls can work. """ if not ('bts_registered' in conf and conf['bts_registered']): logger.error('BTS is not yet registered, skipping VPN setup, killing' ' all services.') for s in SERVICES: if s.name == 'endagad': continue s.stop() return # If the VPN is down, try to start it, then restart FS if we succeed. if not system_utilities.get_vpn_ip(): max_attempts = 10 for _ in range(0, max_attempts): # Sometimes the vpn service is started, but the VPN is still down. # If this is the case, stop the vpn service first. openvpn_service = Service.SystemService('ccm-openvpn') if openvpn_service.status() == ServiceState.Running: openvpn_service.stop() if openvpn_service.start(): logger.notice('VPN service started') if system_utilities.get_vpn_ip(): logger.notice('VPN up - restarting freeswitch') Service.SystemService('freeswitch').restart() else: logger.error('VPN interface (%s) is down' % conf.get('external_interface')) else: logger.error( 'VPN failed to start after registration, retrying.') time.sleep(3) if not system_utilities.get_vpn_ip(): logger.error('Failed to set up VPN after %d attempts!' % max_attempts) # Start all the other services. This is safe to run if services are # already started. for s in SERVICES: try: s.start() except Exception as e: logger.critical("Exception %s while starting %s" % (e, s.name))
def process_config_update(self, data_dict): """Process an endaga settings section in a checkin response. This should be a dictionary of key-value pairs. For each pair, we add it to the ConfigDB. """ for (key, v) in data_dict.items(): if key not in self: logger.notice("Adding endaga setting: %s -> %s" % (key, v)) else: old_v = self[key] if self._ducktype(v) != old_v: logger.notice("Changing endaga setting:" "%s -> %s (was %s)" % (key, v, old_v)) else: continue self[key] = v
def update_vpn(): """ If the BTS is registered, try to start the VPN. If the BTS is not registered, skip. If the BTS is unregistered (on the dashboard), no services are available. Regardless of whether the VPN is up or down, all services should be started (this will enable disconnected operation). However, when the VPN comes online, we need to restart FS to make sure that we're bound to the VPN IP so outgoing calls can work. """ if not ('bts_registered' in conf and conf['bts_registered']): logger.error("BTS is not yet registered, skipping VPN setup, killing" " all services.") for s in SERVICES: if s.name == "endagad": continue s.stop() return # If the VPN is down, try to start it, then restart FS if we succeed. if not system_utilities.get_vpn_ip(): max_attempts = 10 for _ in range(0, max_attempts): # Sometimes the vpn service is started, but the VPN is still down. # If this is the case, stop the vpn service first. openvpn_service = Service.SupervisorService("openvpn") if openvpn_service.status() == ServiceState.Running: openvpn_service.stop() if (openvpn_service.start() and system_utilities.get_vpn_ip()): logger.notice("VPN up restarting services") Service.SystemService("freeswitch").restart() else: logger.error("VPN didn't come up after registration," " retrying.") time.sleep(3) if not system_utilities.get_vpn_ip(): logger.error("Failed to set up VPN after %d attempts!" % max_attempts) # Start all the other services. This is safe to run if services are # already started. for s in SERVICES: s.start()
def run(self): """ Main loop for endagad. This moves the system through the various states of operation -- it should be a state machine really! General flow is: 1) Tries to get configuration from server to produce VPN keys 2) Generates keys locally. 3) Sends CSR for signing, returns that. 4) Starts system services (FS, BTS, etc) and configures them appropriately. Note configuration can change depending on registration and VPN state of the system. 5) Runs checkin periodically. """ eapi = interconnect.endaga_ic(self._conf) if 'registration_interval' not in self._conf: self._conf['registration_interval'] = 60 UNHEALTHY_THRESH = self._conf.get('bts.unhealthy_threshold', 3) unhealthy_count = UNHEALTHY_THRESH # fail quickly on first pass while True: # Retrieve keys/tokens, or do nothing if we have them. logger.notice("Performing gen_keys") registration.generate_keys() # generate_keys() loads auth token on success. Need to update the # interconnect client's token if so. if eapi.token is None: eapi.token = self._conf['endaga_token'] # Try to register/get VPN credentials. Tries forever if fails. logger.notice("Performing register") registration.register(eapi) # Registered, start services and tries to start VPN. Stop # everything otherwise. logger.notice("Performing clear_pid") registration.clear_old_pid() logger.notice("Performing update_vpn") registration.update_vpn() # At this point, all services should be up, so we can perform # additional configuration. self._reset_bts_config() # Update the inbound_url if the VPN is up. if system_utilities.get_vpn_ip() is not None: logger.notice("Performing register_update") registration.register_update(eapi) logger.notice("Performing ensure_fs_external_bound") registration.ensure_fs_external_bound_to_vpn() # Send checkin to cloud try: # Sends events, tries to get config info. Can proceed w/o VPN. logger.notice("Performing checkin.") checkin_data = eapi.checkin(timeout=30) logger.notice("Performing system health check.") if not registration.system_healthcheck(checkin_data): unhealthy_count += 1 logger.notice("System unhealthy: %d" % unhealthy_count) else: unhealthy_count = 0 except (ConnectionError, Timeout): logger.error( "checkin failed due to connection error or timeout.") except BSSError as e: logger.error("bts exception: %s" % e) if unhealthy_count > UNHEALTHY_THRESH: logger.notice("BTS seems unhealthy, restarting BTS services.") bts.restart() Service.SystemService("freeswitch").restart() # Upgrade the endaga metapackage, when appropriate and only if that # feature is enabled. logger.notice("Performing autoupgrade") system_utilities.try_to_autoupgrade() # Sleep for some amount of time before retrying logger.notice("Performing sleep") time.sleep(self._conf['registration_interval'])
def set_factory_config(self): """ Verifies that OpenBTS TRX frequency offset settings are set to factory defaults. This is only necessary for RAD1-based systems. If not set to defaults, we need to update OpenBTS. First, we use the `freqcorr` command to immediately update the offset; then, we set the proper self.configuration value so we will use that in the future (as this is a static variable, it's not enough to just update the self.config). The default value is defined as follows. First, we check the self.configDB for a "RealTRXFreqOffset" key. If this is defined, we use that. If this is not defined, we assume the value listed as default for the TRX.RadioFrequencyOffset key in OpenBTS is correct; this comes from the trxfactory settings burned into the radio. Unfortunately, some radios from Range are not properly calibrated, so we perform that calibration in-house and set the RealTRXFreqOffset flag in our own self.config DB. We also don't have a script implemented to update the contents of the RAD1 EEPROM, so easier to just do this. We also change the radio band if necessary -- there's some weirdness in the implementation described below. If the band is changed, we must restart OpenBTS for those changes to take effect, so we do that here as well. Returns: Whether or not OpenBTS needs to be restarted TODO(shaddi): This does not support non-RAD1 systems, we need to add support for UHD. """ restart_required = False # First set frequency offset -- only needed for RAD1 try: res = self.openbts.read_config("TRX.RadioFrequencyOffset") except TimeoutError as e: logger.error( "Unable to query OpenBTS, can't set factory defaults!") raise BSSError(e) try: default_offset = self.conf['RealTRXFreqOffset'] except KeyError: default_offset = res.data['defaultValue'] if default_offset != res.data['value']: self.openbts.update_config("TRX.RadioFrequencyOffset", default_offset) # We run this command via the CLI to immediately update the frequency # offset. r = delegator.run("/OpenBTS/OpenBTSCLI -c 'freqcorr %s'" % (default_offset, ), timeout=self.conf['bss_timeout']) if r.return_code != 0: err = "Error %s: %s" % (r.return_code, " ".join(r.cmd)) logger.error(err) raise BSSError(err) logger.notice("Frequency offset update to %s" % default_offset) # Set band res = self.openbts.read_config("GSM.Radio.Band") if res.data['defaultValue'] != res.data['value']: # We use delegator to update this value instead of the NodeManager # interface because of a weird behavior in self.openbts. We rely on OpenBTS # to report what ARFCNs in the band it supports when we read the self.config # of GSM.Radio.C0 (specifically the validValues section). If we update # the self.config via NodeManager, the GSM.Radio.Band self.config setting would # be applied, but then reading the value of GSM.Radio.C0 will still # return the previous band's set of valid values! For whatever reason, # updating the band setting from the CLI will give us the valid ARFCNs # for the new band setting. logger.notice("Trying to set radio band from %s to %s" % (res.data['value'], res.data['defaultValue'])) r = delegator.run( "/OpenBTS/OpenBTSCLI -c 'config GSM.Radio.Band %s'" % (res.data['defaultValue'], ), timeout=self.conf['bss_timeout']) if r.return_code != 0: err = "Error %s: %s" % (r.return_code, " ".join(r.cmd)) logger.error(err) raise BSSError(err) restart_required = True logger.notice("Updated radio band to %s" % res.data['defaultValue']) # Set ARFCN to lowest for the band res = self.openbts.read_config("GSM.Radio.C0") valid_arfcns = [_.split("|")[0] \ for _ in res.data['validValues'].split(",")] if valid_arfcns[0] != res.data['value']: logger.notice("Trying to set radio ARFCN from %s to %s" % (res.data['value'], valid_arfcns[0])) self.openbts.update_config("GSM.Radio.C0", valid_arfcns[0]) logger.notice("Updated ARFCN to %s" % valid_arfcns[0]) return True return restart_required
def checkin_worker(self): logger.notice("Endaga: Checkin requested by dashboard.") self.ic.checkin()
""" Direct logging output to stdout during testing. Copyright (c) 2016-present, Facebook, Inc. All rights reserved. This source code is licensed under the BSD-style license found in the LICENSE file in the root directory of this source tree. An additional grant of patent rights can be found in the PATENTS file in the same directory. """ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals from logging import StreamHandler, DEBUG # use stdout for output since nosetests will swallow it from sys import stdout from ccm.common.logger import DefaultLogger, notice DefaultLogger.update_handler(StreamHandler(stdout), DEBUG) notice("directing logger output to stdout during testing")
def process_prices(pricing_data, config_db): """Processes incoming price data from the checkin response. Saves these prices in the ConfigDB, creating keys if needed. Args: pricing_data: a list of dicts. Each dict is either an on-network receive tier, an on-network send tier, an off-network receive tier or an off-network send tier. There are many off-network send tiers, one for each prefix (country). But there is only one of the other three classes of tiers. All costs are given in millicents. For example: { 'directionality': 'off_network_send', 'prefix': '53', 'country_name': 'Finland', 'country_code': 'FI', 'cost_to_subscriber_per_sms': 5000, 'cost_to_subscriber_per_min': 2000, 'billable_unit': 1, }, { 'directionality': 'off_network_receive', 'cost_to_subscriber_per_sms': 100, 'cost_to_subscriber_per_min': 200, 'billable_unit': 1, }, { 'directionality': 'on_network_send', 'cost_to_subscriber_per_sms': 25, 'cost_to_subscriber_per_min': 50, 'billable_unit': 1, }, { 'directionality': 'on_network_receive', 'cost_to_subscriber_per_sms': 10, 'cost_to_subscriber_per_min': 20, 'billable_unit': 1, } """ for price_group in pricing_data: # Set the config db keys. prefixless_keys = ('off_network_receive', 'on_network_receive', 'on_network_send') if price_group['directionality'] in prefixless_keys: sms_key = ('prices.%s.cost_to_subscriber_per_sms' % (price_group['directionality'], )) call_key = ('prices.%s.cost_to_subscriber_per_min' % (price_group['directionality'], )) billable_unit_key = ('prices.%s.billable_unit' % (price_group['directionality'], )) elif price_group['directionality'] == 'off_network_send': sms_key = ('prices.%s.%s.cost_to_subscriber_per_sms' % ('off_network_send', price_group['prefix'])) call_key = ('prices.%s.%s.cost_to_subscriber_per_min' % ('off_network_send', price_group['prefix'])) billable_unit_key = ('prices.%s.%s.billable_unit' % ('off_network_send', price_group['prefix'])) # Get the actual values specified in the checkin response data. sms_new_value = price_group['cost_to_subscriber_per_sms'] call_new_value = price_group['cost_to_subscriber_per_min'] # Legacy cloud responses may not include this, # so we default to 1 billable_unit_new_val = price_group.get('billable_unit', 1) # Add to the config db if necessary. for key, new_value in [ (sms_key, sms_new_value), (call_key, call_new_value), (billable_unit_key, billable_unit_new_val)]: old_value = config_db.get(key) if old_value is None: logger.notice("adding key: %s -> %s" % (key, new_value)) config_db[key] = new_value else: if config_db._ducktype(new_value) != old_value: logger.notice("changing key: %s -> %s (was %s)" % (key, new_value, old_value)) config_db[key] = new_value