def update_vpn():
"""
If the BTS is registered, try to start the VPN. If the BTS is not
registered, skip.
If the BTS is unregistered (on the dashboard), no services are available.
Regardless of whether the VPN is up or down, all services should be started
(this will enable disconnected operation). However, when the VPN comes
online, we need to restart FS to make sure that we're bound to the VPN IP
so outgoing calls can work.
"""
if not ('bts_registered' in conf and conf['bts_registered']):
logger.error('BTS is not yet registered, skipping VPN setup, killing'
' all services.')
for s in SERVICES:
if s.name == 'endagad':
continue
s.stop()
return
# If the VPN is down, try to start it, then restart FS if we succeed.
if not system_utilities.get_vpn_ip():
max_attempts = 10
for _ in range(0, max_attempts):
# Sometimes the vpn service is started, but the VPN is still down.
# If this is the case, stop the vpn service first.
openvpn_service = Service.SystemService('ccm-openvpn')
if openvpn_service.status() == ServiceState.Running:
openvpn_service.stop()
if openvpn_service.start():
logger.notice('VPN service started')
if system_utilities.get_vpn_ip():
logger.notice('VPN up - restarting freeswitch')
Service.SystemService('freeswitch').restart()
else:
logger.error('VPN interface (%s) is down' %
conf.get('external_interface'))
else:
logger.error(
'VPN failed to start after registration, retrying.')
time.sleep(3)
if not system_utilities.get_vpn_ip():
logger.error('Failed to set up VPN after %d attempts!' %
max_attempts)
# Start all the other services. This is safe to run if services are
# already started.
for s in SERVICES:
try:
s.start()
except Exception as e:
logger.critical("Exception %s while starting %s" % (e, s.name))
def register_update(eapi):
"""Ensures the inbound URL for the BTS is up to date."""
vpn_ip = system_utilities.get_vpn_ip()
vpn_status = "up" if vpn_ip else "down"
# This could fail when offline! Must handle connection exceptions.
try:
params = {
'bts_uuid': snowflake.snowflake(),
'vpn_status': vpn_status,
'vpn_ip': vpn_ip,
# federer always runs on port 80, but didn't in old versions
'federer_port': "80",
}
r = requests.get(conf['registry'] + "/bts/register",
params=params,
headers=eapi.auth_header,
timeout=11)
if r.status_code == 200:
try:
d = json.loads(r.text)
if 'bts_secret' in d:
conf['bts_secret'] = d['bts_secret']
except ValueError:
pass
return r.text
else:
raise ValueError("BTS registration update failed with status"
" %d (%s)" % (r.status_code, r.text))
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
logger.error("register_update failed due to connection error or"
" timeout.")
def update_vpn():
"""
If the BTS is registered, try to start the VPN. If the BTS is not
registered, skip.
If the BTS is unregistered (on the dashboard), no services are available.
Regardless of whether the VPN is up or down, all services should be started
(this will enable disconnected operation). However, when the VPN comes
online, we need to restart FS to make sure that we're bound to the VPN IP
so outgoing calls can work.
"""
if not ('bts_registered' in conf and conf['bts_registered']):
logger.error("BTS is not yet registered, skipping VPN setup, killing"
" all services.")
for s in SERVICES:
if s.name == "endagad":
continue
s.stop()
return
# If the VPN is down, try to start it, then restart FS if we succeed.
if not system_utilities.get_vpn_ip():
max_attempts = 10
for _ in range(0, max_attempts):
# Sometimes the vpn service is started, but the VPN is still down.
# If this is the case, stop the vpn service first.
openvpn_service = Service.SupervisorService("openvpn")
if openvpn_service.status() == ServiceState.Running:
openvpn_service.stop()
if (openvpn_service.start() and system_utilities.get_vpn_ip()):
logger.notice("VPN up restarting services")
Service.SystemService("freeswitch").restart()
else:
logger.error("VPN didn't come up after registration,"
" retrying.")
time.sleep(3)
if not system_utilities.get_vpn_ip():
logger.error("Failed to set up VPN after %d attempts!" %
max_attempts)
# Start all the other services. This is safe to run if services are
# already started.
for s in SERVICES:
s.start()
def ensure_fs_external_bound_to_vpn():
# Make sure that we're bound to the VPN IP on the external sofia profile,
# assuming the VPN is up.
vpn_ip = system_utilities.get_vpn_ip()
if not vpn_ip:
return
external_profile_ip = system_utilities.get_fs_profile_ip('external')
if external_profile_ip != vpn_ip: # TODO: treat these as netaddr, not string
logger.warning('external profile should be bound to VPN IP and isn\'t,'
' restarting FS.')
Service.SystemService('freeswitch').restart()
def register_update(eapi):
"""Ensures the inbound URL for the BTS is up to date."""
vpn_ip = system_utilities.get_vpn_ip()
vpn_status = 'up' if vpn_ip else 'down'
# This could fail when offline! Must handle connection exceptions.
params = {
'bts_uuid': _get_snowflake(),
'vpn_status': vpn_status,
'vpn_ip': vpn_ip,
'federer_port': '80',
}
try:
d = _send_cloud_req(requests.get,
'/bts/register',
'BTS registration',
params=params,
headers=eapi.auth_header,
timeout=11)
if 'bts_secret' in d:
conf['bts_secret'] = d['bts_secret']
except RegistrationError as ex:
logger.error(str(ex))
def run(self):
"""
Main loop for endagad. This moves the system through the various
states of operation -- it should be a state machine really!
General flow is:
1) Tries to get configuration from server to produce VPN keys
2) Generates keys locally.
3) Sends CSR for signing, returns that.
4) Starts system services (FS, BTS, etc) and configures them
appropriately. Note configuration can change depending on registration
and VPN state of the system.
5) Runs checkin periodically.
"""
eapi = interconnect.endaga_ic(self._conf)
if 'registration_interval' not in self._conf:
self._conf['registration_interval'] = 60
UNHEALTHY_THRESH = self._conf.get('bts.unhealthy_threshold', 3)
unhealthy_count = UNHEALTHY_THRESH # fail quickly on first pass
while True:
# Retrieve keys/tokens, or do nothing if we have them.
logger.notice("Performing gen_keys")
registration.generate_keys()
# generate_keys() loads auth token on success. Need to update the
# interconnect client's token if so.
if eapi.token is None:
eapi.token = self._conf['endaga_token']
# Try to register/get VPN credentials. Tries forever if fails.
logger.notice("Performing register")
registration.register(eapi)
# Registered, start services and tries to start VPN. Stop
# everything otherwise.
logger.notice("Performing clear_pid")
registration.clear_old_pid()
logger.notice("Performing update_vpn")
registration.update_vpn()
# At this point, all services should be up, so we can perform
# additional configuration.
self._reset_bts_config()
# Update the inbound_url if the VPN is up.
if system_utilities.get_vpn_ip() is not None:
logger.notice("Performing register_update")
registration.register_update(eapi)
logger.notice("Performing ensure_fs_external_bound")
registration.ensure_fs_external_bound_to_vpn()
# Send checkin to cloud
try:
# Sends events, tries to get config info. Can proceed w/o VPN.
logger.notice("Performing checkin.")
checkin_data = eapi.checkin(timeout=30)
logger.notice("Performing system health check.")
if not registration.system_healthcheck(checkin_data):
unhealthy_count += 1
logger.notice("System unhealthy: %d" % unhealthy_count)
else:
unhealthy_count = 0
except (ConnectionError, Timeout):
logger.error(
"checkin failed due to connection error or timeout.")
except BSSError as e:
logger.error("bts exception: %s" % e)
if unhealthy_count > UNHEALTHY_THRESH:
logger.notice("BTS seems unhealthy, restarting BTS services.")
bts.restart()
Service.SystemService("freeswitch").restart()
# Upgrade the endaga metapackage, when appropriate and only if that
# feature is enabled.
logger.notice("Performing autoupgrade")
system_utilities.try_to_autoupgrade()
# Sleep for some amount of time before retrying
logger.notice("Performing sleep")
time.sleep(self._conf['registration_interval'])