def upgrade_endaga(channel):
"""Upgrades the endaga metapackage."""
# Validate.
if channel not in ('stable', 'beta'):
logger.error('cannot upgrade to the "%s" channel' % channel)
return
logger.notice('upgrading the endaga metapackage with channel %s' % channel)
# Update packages.
response = delegator.run('sudo apt-get update')
if response.return_code != 0:
message = 'Error while running "apt-get update": %s' % response.out
logger.error(message)
# Try a dry-run of the upgrade.
command = ('sudo apt-get install --assume-yes --dry-run'
' --only-upgrade -t %s endaga' % channel)
response = delegator.run(command)
if response.return_code != 0:
message = ('Error while dry running the endaga upgrade: %s' %
response.out)
logger.error(message)
return
# Upgrade just the metapackage.
command = ('sudo apt-get install --assume-yes'
' --only-upgrade -t %s endaga' % channel)
response = delegator.run(command)
if response.return_code != 0:
message = 'Error while upgrading endaga: %s' % response.out
logger.error(message)
def req_log(self, data):
PERMITTED_LOGS = ['endaga', 'syslog']
required_fields = ["start", "end", "log_name"]
if not all([_ in data for _ in required_fields]):
return web.BadRequest()
# By default there are no window start or end
window_start = None
window_end = None
try:
window_start = dateutil.parser.parse(data['start'])
except ValueError:
pass
try:
window_end = dateutil.parser.parse(data['end'])
except ValueError:
pass
if data['log_name'] not in PERMITTED_LOGS:
return web.BadRequest()
logger.notice(
"Log %s requested by dashboard from %s to %s" %
(data['log_name'], window_start or '-', window_end or '-'))
t = threading.Thread(target=self.log_worker,
args=(data['msgid'], window_start, window_end,
data['log_name']))
t.start()
return web.Accepted()
def __init__(self):
self._conf = ConfigDB()
# initialise logging level from DB (if set - otherwise 'warning')
# NB - this is the ONLY time changes to the log level are actually
# passed to the logging framework
log_level = self._conf.get("logger.global.log_level", "warning")
logger.DefaultLogger.update_handler(level=log_level)
logger.notice("EndagaD started")
def process_update(self, net_subs):
"""
Processes the subscriber list. Format is:
{
IMSI1: {'number': [<numbers>,...], 'balance': {<PN counter>}},
IMSI2: ...
}
This updates the BTS with all subscribers instructed by the cloud; any
subscribers that are not reported by the cloud will be removed from
this BTS.
"""
# dict where keys are imsis and values are sub info
bts_imsis = self.get_subscriber_imsis()
net_imsis = set(net_subs.keys())
subs_to_add = net_imsis.difference(bts_imsis)
subs_to_delete = bts_imsis.difference(net_imsis)
subs_to_update = bts_imsis.intersection(net_imsis)
for imsi in subs_to_delete:
self.delete_subscriber(imsi)
# TODO(shasan) does not add new numbers
for imsi in subs_to_update:
sub = net_subs[imsi]
try:
bal = crdt.PNCounter.from_state(sub['balance'])
self.update_balance(imsi, bal)
except SubscriberNotFound as e:
logger.warning(
"Balance sync fail! IMSI: %s is not found Error: %s" %
(imsi, e))
except ValueError as e:
logger.error("Balance sync fail! IMSI: %s, %s Error: %s" %
(imsi, sub['balance'], e))
subs_to_add.add(imsi) # try to add it (again)
for imsi in subs_to_add:
sub = net_subs[imsi]
numbers = sub['numbers']
if not numbers:
logger.notice("IMSI with no numbers? %s" % imsi)
continue
self.create_subscriber(imsi, numbers[0])
for n in numbers[1:]:
self.add_number(imsi, n)
try:
bal = crdt.PNCounter.from_state(sub['balance'])
self.update_balance(imsi, bal)
except (SubscriberNotFound, ValueError) as e:
logger.error("Balance sync fail! IMSI: %s, %s Error: %s" %
(imsi, sub['balance'], e))
def _reset_bts_config(self):
logger.notice("Performing set_factory")
try:
if bts.set_factory_config():
logger.notice("Restarting BTS")
bts.restart()
Service.SystemService("freeswitch").restart()
except BSSError as e:
logger.error("bts is probably down: %s" % e)
except Exception as e:
# OSError, IOError or whatever envoy will raise
logger.critical("something unexpected happened: %s" % e)
def update_vpn():
"""
If the BTS is registered, try to start the VPN. If the BTS is not
registered, skip.
If the BTS is unregistered (on the dashboard), no services are available.
Regardless of whether the VPN is up or down, all services should be started
(this will enable disconnected operation). However, when the VPN comes
online, we need to restart FS to make sure that we're bound to the VPN IP
so outgoing calls can work.
"""
if not ('bts_registered' in conf and conf['bts_registered']):
logger.error('BTS is not yet registered, skipping VPN setup, killing'
' all services.')
for s in SERVICES:
if s.name == 'endagad':
continue
s.stop()
return
# If the VPN is down, try to start it, then restart FS if we succeed.
if not system_utilities.get_vpn_ip():
max_attempts = 10
for _ in range(0, max_attempts):
# Sometimes the vpn service is started, but the VPN is still down.
# If this is the case, stop the vpn service first.
openvpn_service = Service.SystemService('ccm-openvpn')
if openvpn_service.status() == ServiceState.Running:
openvpn_service.stop()
if openvpn_service.start():
logger.notice('VPN service started')
if system_utilities.get_vpn_ip():
logger.notice('VPN up - restarting freeswitch')
Service.SystemService('freeswitch').restart()
else:
logger.error('VPN interface (%s) is down' %
conf.get('external_interface'))
else:
logger.error(
'VPN failed to start after registration, retrying.')
time.sleep(3)
if not system_utilities.get_vpn_ip():
logger.error('Failed to set up VPN after %d attempts!' %
max_attempts)
# Start all the other services. This is safe to run if services are
# already started.
for s in SERVICES:
try:
s.start()
except Exception as e:
logger.critical("Exception %s while starting %s" % (e, s.name))
def process_config_update(self, data_dict):
"""Process an endaga settings section in a checkin response.
This should be a dictionary of key-value pairs. For each pair, we add
it to the ConfigDB.
"""
for (key, v) in data_dict.items():
if key not in self:
logger.notice("Adding endaga setting: %s -> %s" % (key, v))
else:
old_v = self[key]
if self._ducktype(v) != old_v:
logger.notice("Changing endaga setting:"
"%s -> %s (was %s)" % (key, v, old_v))
else:
continue
self[key] = v
def update_vpn():
"""
If the BTS is registered, try to start the VPN. If the BTS is not
registered, skip.
If the BTS is unregistered (on the dashboard), no services are available.
Regardless of whether the VPN is up or down, all services should be started
(this will enable disconnected operation). However, when the VPN comes
online, we need to restart FS to make sure that we're bound to the VPN IP
so outgoing calls can work.
"""
if not ('bts_registered' in conf and conf['bts_registered']):
logger.error("BTS is not yet registered, skipping VPN setup, killing"
" all services.")
for s in SERVICES:
if s.name == "endagad":
continue
s.stop()
return
# If the VPN is down, try to start it, then restart FS if we succeed.
if not system_utilities.get_vpn_ip():
max_attempts = 10
for _ in range(0, max_attempts):
# Sometimes the vpn service is started, but the VPN is still down.
# If this is the case, stop the vpn service first.
openvpn_service = Service.SupervisorService("openvpn")
if openvpn_service.status() == ServiceState.Running:
openvpn_service.stop()
if (openvpn_service.start() and system_utilities.get_vpn_ip()):
logger.notice("VPN up restarting services")
Service.SystemService("freeswitch").restart()
else:
logger.error("VPN didn't come up after registration,"
" retrying.")
time.sleep(3)
if not system_utilities.get_vpn_ip():
logger.error("Failed to set up VPN after %d attempts!" %
max_attempts)
# Start all the other services. This is safe to run if services are
# already started.
for s in SERVICES:
s.start()
def run(self):
"""
Main loop for endagad. This moves the system through the various
states of operation -- it should be a state machine really!
General flow is:
1) Tries to get configuration from server to produce VPN keys
2) Generates keys locally.
3) Sends CSR for signing, returns that.
4) Starts system services (FS, BTS, etc) and configures them
appropriately. Note configuration can change depending on registration
and VPN state of the system.
5) Runs checkin periodically.
"""
eapi = interconnect.endaga_ic(self._conf)
if 'registration_interval' not in self._conf:
self._conf['registration_interval'] = 60
UNHEALTHY_THRESH = self._conf.get('bts.unhealthy_threshold', 3)
unhealthy_count = UNHEALTHY_THRESH # fail quickly on first pass
while True:
# Retrieve keys/tokens, or do nothing if we have them.
logger.notice("Performing gen_keys")
registration.generate_keys()
# generate_keys() loads auth token on success. Need to update the
# interconnect client's token if so.
if eapi.token is None:
eapi.token = self._conf['endaga_token']
# Try to register/get VPN credentials. Tries forever if fails.
logger.notice("Performing register")
registration.register(eapi)
# Registered, start services and tries to start VPN. Stop
# everything otherwise.
logger.notice("Performing clear_pid")
registration.clear_old_pid()
logger.notice("Performing update_vpn")
registration.update_vpn()
# At this point, all services should be up, so we can perform
# additional configuration.
self._reset_bts_config()
# Update the inbound_url if the VPN is up.
if system_utilities.get_vpn_ip() is not None:
logger.notice("Performing register_update")
registration.register_update(eapi)
logger.notice("Performing ensure_fs_external_bound")
registration.ensure_fs_external_bound_to_vpn()
# Send checkin to cloud
try:
# Sends events, tries to get config info. Can proceed w/o VPN.
logger.notice("Performing checkin.")
checkin_data = eapi.checkin(timeout=30)
logger.notice("Performing system health check.")
if not registration.system_healthcheck(checkin_data):
unhealthy_count += 1
logger.notice("System unhealthy: %d" % unhealthy_count)
else:
unhealthy_count = 0
except (ConnectionError, Timeout):
logger.error(
"checkin failed due to connection error or timeout.")
except BSSError as e:
logger.error("bts exception: %s" % e)
if unhealthy_count > UNHEALTHY_THRESH:
logger.notice("BTS seems unhealthy, restarting BTS services.")
bts.restart()
Service.SystemService("freeswitch").restart()
# Upgrade the endaga metapackage, when appropriate and only if that
# feature is enabled.
logger.notice("Performing autoupgrade")
system_utilities.try_to_autoupgrade()
# Sleep for some amount of time before retrying
logger.notice("Performing sleep")
time.sleep(self._conf['registration_interval'])
def set_factory_config(self):
"""
Verifies that OpenBTS TRX frequency offset settings are set to factory
defaults. This is only necessary for RAD1-based systems.
If not set to defaults, we need to update OpenBTS. First, we use the
`freqcorr` command to immediately update the offset; then, we set the
proper self.configuration value so we will use that in the future (as this is a
static variable, it's not enough to just update the self.config).
The default value is defined as follows. First, we check the self.configDB for a
"RealTRXFreqOffset" key. If this is defined, we use that. If this is
not defined, we assume the value listed as default for the
TRX.RadioFrequencyOffset key in OpenBTS is correct; this comes from the
trxfactory settings burned into the radio. Unfortunately, some radios from
Range are not properly calibrated, so we perform that calibration in-house
and set the RealTRXFreqOffset flag in our own self.config DB. We also don't have
a script implemented to update the contents of the RAD1 EEPROM, so easier
to just do this.
We also change the radio band if necessary -- there's some weirdness in the
implementation described below. If the band is changed, we must restart
OpenBTS for those changes to take effect, so we do that here as well.
Returns:
Whether or not OpenBTS needs to be restarted
TODO(shaddi): This does not support non-RAD1 systems, we need to add
support for UHD.
"""
restart_required = False
# First set frequency offset -- only needed for RAD1
try:
res = self.openbts.read_config("TRX.RadioFrequencyOffset")
except TimeoutError as e:
logger.error(
"Unable to query OpenBTS, can't set factory defaults!")
raise BSSError(e)
try:
default_offset = self.conf['RealTRXFreqOffset']
except KeyError:
default_offset = res.data['defaultValue']
if default_offset != res.data['value']:
self.openbts.update_config("TRX.RadioFrequencyOffset",
default_offset)
# We run this command via the CLI to immediately update the frequency
# offset.
r = delegator.run("/OpenBTS/OpenBTSCLI -c 'freqcorr %s'" %
(default_offset, ),
timeout=self.conf['bss_timeout'])
if r.return_code != 0:
err = "Error %s: %s" % (r.return_code, " ".join(r.cmd))
logger.error(err)
raise BSSError(err)
logger.notice("Frequency offset update to %s" % default_offset)
# Set band
res = self.openbts.read_config("GSM.Radio.Band")
if res.data['defaultValue'] != res.data['value']:
# We use delegator to update this value instead of the NodeManager
# interface because of a weird behavior in self.openbts. We rely on OpenBTS
# to report what ARFCNs in the band it supports when we read the self.config
# of GSM.Radio.C0 (specifically the validValues section). If we update
# the self.config via NodeManager, the GSM.Radio.Band self.config setting would
# be applied, but then reading the value of GSM.Radio.C0 will still
# return the previous band's set of valid values! For whatever reason,
# updating the band setting from the CLI will give us the valid ARFCNs
# for the new band setting.
logger.notice("Trying to set radio band from %s to %s" %
(res.data['value'], res.data['defaultValue']))
r = delegator.run(
"/OpenBTS/OpenBTSCLI -c 'config GSM.Radio.Band %s'" %
(res.data['defaultValue'], ),
timeout=self.conf['bss_timeout'])
if r.return_code != 0:
err = "Error %s: %s" % (r.return_code, " ".join(r.cmd))
logger.error(err)
raise BSSError(err)
restart_required = True
logger.notice("Updated radio band to %s" %
res.data['defaultValue'])
# Set ARFCN to lowest for the band
res = self.openbts.read_config("GSM.Radio.C0")
valid_arfcns = [_.split("|")[0] \
for _ in res.data['validValues'].split(",")]
if valid_arfcns[0] != res.data['value']:
logger.notice("Trying to set radio ARFCN from %s to %s" %
(res.data['value'], valid_arfcns[0]))
self.openbts.update_config("GSM.Radio.C0", valid_arfcns[0])
logger.notice("Updated ARFCN to %s" % valid_arfcns[0])
return True
return restart_required
def checkin_worker(self):
logger.notice("Endaga: Checkin requested by dashboard.")
self.ic.checkin()
""" Direct logging output to stdout during testing.
Copyright (c) 2016-present, Facebook, Inc.
All rights reserved.
This source code is licensed under the BSD-style license found in the
LICENSE file in the root directory of this source tree. An additional grant
of patent rights can be found in the PATENTS file in the same directory.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from logging import StreamHandler, DEBUG
# use stdout for output since nosetests will swallow it
from sys import stdout
from ccm.common.logger import DefaultLogger, notice
DefaultLogger.update_handler(StreamHandler(stdout), DEBUG)
notice("directing logger output to stdout during testing")
def process_prices(pricing_data, config_db):
"""Processes incoming price data from the checkin response.
Saves these prices in the ConfigDB, creating keys if needed.
Args:
pricing_data: a list of dicts. Each dict is either an on-network
receive tier, an on-network send tier, an off-network receive
tier or an off-network send tier. There are many off-network
send tiers, one for each prefix (country). But there is only one
of the other three classes of tiers. All costs are given in
millicents. For example:
{
'directionality': 'off_network_send',
'prefix': '53',
'country_name': 'Finland',
'country_code': 'FI',
'cost_to_subscriber_per_sms': 5000,
'cost_to_subscriber_per_min': 2000,
'billable_unit': 1,
}, {
'directionality': 'off_network_receive',
'cost_to_subscriber_per_sms': 100,
'cost_to_subscriber_per_min': 200,
'billable_unit': 1,
}, {
'directionality': 'on_network_send',
'cost_to_subscriber_per_sms': 25,
'cost_to_subscriber_per_min': 50,
'billable_unit': 1,
}, {
'directionality': 'on_network_receive',
'cost_to_subscriber_per_sms': 10,
'cost_to_subscriber_per_min': 20,
'billable_unit': 1,
}
"""
for price_group in pricing_data:
# Set the config db keys.
prefixless_keys = ('off_network_receive', 'on_network_receive',
'on_network_send')
if price_group['directionality'] in prefixless_keys:
sms_key = ('prices.%s.cost_to_subscriber_per_sms' %
(price_group['directionality'], ))
call_key = ('prices.%s.cost_to_subscriber_per_min' %
(price_group['directionality'], ))
billable_unit_key = ('prices.%s.billable_unit' %
(price_group['directionality'], ))
elif price_group['directionality'] == 'off_network_send':
sms_key = ('prices.%s.%s.cost_to_subscriber_per_sms' %
('off_network_send', price_group['prefix']))
call_key = ('prices.%s.%s.cost_to_subscriber_per_min' %
('off_network_send', price_group['prefix']))
billable_unit_key = ('prices.%s.%s.billable_unit' %
('off_network_send', price_group['prefix']))
# Get the actual values specified in the checkin response data.
sms_new_value = price_group['cost_to_subscriber_per_sms']
call_new_value = price_group['cost_to_subscriber_per_min']
# Legacy cloud responses may not include this,
# so we default to 1
billable_unit_new_val = price_group.get('billable_unit', 1)
# Add to the config db if necessary.
for key, new_value in [
(sms_key, sms_new_value),
(call_key, call_new_value),
(billable_unit_key, billable_unit_new_val)]:
old_value = config_db.get(key)
if old_value is None:
logger.notice("adding key: %s -> %s" %
(key, new_value))
config_db[key] = new_value
else:
if config_db._ducktype(new_value) != old_value:
logger.notice("changing key: %s -> %s (was %s)" %
(key, new_value, old_value))
config_db[key] = new_value
