def register_update(eapi):
"""Ensures the inbound URL for the BTS is up to date."""
vpn_ip = system_utilities.get_vpn_ip()
vpn_status = "up" if vpn_ip else "down"
# This could fail when offline! Must handle connection exceptions.
try:
params = {
'bts_uuid': snowflake.snowflake(),
'vpn_status': vpn_status,
'vpn_ip': vpn_ip,
# federer always runs on port 80, but didn't in old versions
'federer_port': "80",
}
r = requests.get(conf['registry'] + "/bts/register",
params=params,
headers=eapi.auth_header,
timeout=11)
if r.status_code == 200:
try:
d = json.loads(r.text)
if 'bts_secret' in d:
conf['bts_secret'] = d['bts_secret']
except ValueError:
pass
return r.text
else:
raise ValueError("BTS registration update failed with status"
" %d (%s)" % (r.status_code, r.text))
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
logger.error("register_update failed due to connection error or"
" timeout.")
def send(self, to, from_, body, to_country=None, from_country=None):
"""Send an SMS to our cloud API.
Args:
message params
Returns:
True if the message was accepted, False otherwise.
"""
# Convert "to" to e.164 format. We always add a plus, and
# libphonenumber is smart enough to sort it out from there (even if
# there's already a plus).
message = {
'from': from_,
'to': number_utilities.convert_to_e164("+" + to, None),
'body': body
}
# TODO(matt): use urlparse.urljoin here?
endpoint = self.conf['registry'] + "/send/"
try:
r = requests.post(endpoint, headers=self.auth_header, data=message)
except BaseException as e: # log and rethrow as it was before
logger.error("Endaga: Send SMS network error: %s." % e)
raise
return r.status_code == 202
def register_subscriber(self, imsi):
"""Send a request to the registry server with this BTS unique ID and
the number.
Raises: ValueError if the API failed to register the user
400 - Bad parameters
403 - User is not associated with this BTS
404 - No numbers available
409 - IMSI already registered to another network
500 - Uh-oh
"""
url = self.conf['registry'] + "/register/"
try:
r = requests.post(url,
headers=self.auth_header,
data={
'imsi': imsi,
'bts_uuid': snowflake.snowflake()
})
except BaseException as e: # log and rethrow
logger.error("Endaga: Register network error: %s." % e)
raise
if r.status_code != 200:
raise ValueError(r.text)
return json.loads(r.text)
def log_worker(self, msgid, window_start, window_end, log_name):
logger.info("Log req %s started" % msgid)
log_path = "/var/log/%s" % log_name
tmp_file = ('/tmp/%s-%s.log.gz' % (log_name, msgid))
with gzip.open(tmp_file, 'w') as f:
for msg in log_stream(log_path, window_start, window_end):
f.write(msg)
params = {'msgid': msgid, 'log_name': log_name}
files = {'file': open(tmp_file, 'rb')}
r = requests.post(self.conf['registry'] + "/bts/logfile",
data=params,
files=files,
headers=self.ic.auth_header)
try:
if r.status_code == 200:
logger.info("Log req %s posted successfully" % msgid)
else:
logger.error("Log req %s responded with %s: %s" %
(msgid, r.status_code, r.text))
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as e:
logger.error("Log req %s failed: %s" % (msgid, e))
os.unlink(tmp_file)
def process_bts_settings(self, data_dict):
"""Process bts settings.
TODO: We should revisit how configs are stored on cloud
"""
settings_map = {
'GSM.Identity.MCC':
{'get': self.get_mcc,
'set': self.set_mcc},
'GSM.Identity.MNC':
{'get': self.get_mnc,
'set': self.set_mnc},
'GSM.Identity.ShortName':
{'get': self.get_short_name,
'set': self.set_short_name},
'Control.LUR.OpenRegistration':
{'get': self.get_open_registration,
'set': self.set_open_registration},
'GSM.Radio.C0':
{'get': self.get_arfcn_c0,
'set': self.set_arfcn_c0},
'GSM.Timer.T3212':
{'get': lambda: self.get_timer('3212'),
'set': lambda minutes: self.set_timer('3212', minutes)},
}
for (key, val) in data_dict.items():
try:
cur_val = settings_map[key]['get']()
if cur_val != val:
settings_map[key]['set'](val)
logger.info("Changed bts setting: %s = %s (was %s)" %
(key, val, cur_val))
except Exception as e:
logger.error("Failed to process openbts setting"
"(%s): %s = %s" % (e, key, val))
def GET(self, command):
"""Handles get requests for certain commands."""
valid_get_commands = ('req_usage', 'req_log', 'add_credit',
'req_checkin')
if command not in valid_get_commands:
return web.NotFound()
d = web.input()
if 'jwt' not in d:
return web.BadRequest()
try:
data = self.check_signed_params(d['jwt'])
except ValueError as e:
logger.error("Value error dispatching %s" % (command, ))
return web.BadRequest(str(e))
except Exception as e:
logger.error("Other error dispatching %s: %s" % (command, str(e)))
raise
if command == "req_usage": # NOTE: deprecated 2014oct23
return self.req_checkin()
if command == "req_log":
return self.req_log(data)
elif command == "add_credit":
return self.adjust_credits(data)
elif command == "req_checkin":
return self.req_checkin()
def check_signed_params(self, jwt_data):
"""Checks a JWT signature and message ID.
Decodes the params, makes sure they pass signature (i.e., are valid),
and then checks that we haven't seen the msgid before.
TODO(matt): refactor as this was copied from federer_handlers.common.
Inheriting from common as before does not work because CI
cannot import ESL, an import that comes from
freeswitch_interconnect.
Raises:
ValueError if there are errors
Returns:
True if everything checks out
"""
s = itsdangerous.JSONWebSignatureSerializer(self.conf['bts_secret'])
try:
data = s.loads(jwt_data)
except itsdangerous.BadSignature:
logger.emergency("Bad signature for request, ignoring.")
raise ValueError("Bad signature")
# Make sure the msg hasn't been seen before, if so, discard it.
if "msgid" in data:
if self.msgid_db.seen(str(data['msgid'])):
logger.error("Endaga: Repeat msgid: %s" % (data['msgid'], ))
raise ValueError("Repeat msgid: %s" % (data['msgid'], ))
else:
logger.error("Endaga: No message ID.")
raise ValueError("No message ID.")
return data
def process_update(self, net_subs):
"""
Processes the subscriber list. Format is:
{
IMSI1: {'number': [<numbers>,...], 'balance': {<PN counter>}},
IMSI2: ...
}
This updates the BTS with all subscribers instructed by the cloud; any
subscribers that are not reported by the cloud will be removed from
this BTS.
"""
# dict where keys are imsis and values are sub info
bts_imsis = self.get_subscriber_imsis()
net_imsis = set(net_subs.keys())
subs_to_add = net_imsis.difference(bts_imsis)
subs_to_delete = bts_imsis.difference(net_imsis)
subs_to_update = bts_imsis.intersection(net_imsis)
for imsi in subs_to_delete:
self.delete_subscriber(imsi)
# TODO(shasan) does not add new numbers
for imsi in subs_to_update:
sub = net_subs[imsi]
try:
bal = crdt.PNCounter.from_state(sub['balance'])
self.update_balance(imsi, bal)
except SubscriberNotFound as e:
logger.warning(
"Balance sync fail! IMSI: %s is not found Error: %s" %
(imsi, e))
except ValueError as e:
logger.error("Balance sync fail! IMSI: %s, %s Error: %s" %
(imsi, sub['balance'], e))
subs_to_add.add(imsi) # try to add it (again)
for imsi in subs_to_add:
sub = net_subs[imsi]
numbers = sub['numbers']
if not numbers:
logger.notice("IMSI with no numbers? %s" % imsi)
continue
self.create_subscriber(imsi, numbers[0])
for n in numbers[1:]:
self.add_number(imsi, n)
try:
bal = crdt.PNCounter.from_state(sub['balance'])
self.update_balance(imsi, bal)
except (SubscriberNotFound, ValueError) as e:
logger.error("Balance sync fail! IMSI: %s, %s Error: %s" %
(imsi, sub['balance'], e))
def _reset_bts_config(self):
logger.notice("Performing set_factory")
try:
if bts.set_factory_config():
logger.notice("Restarting BTS")
bts.restart()
Service.SystemService("freeswitch").restart()
except BSSError as e:
logger.error("bts is probably down: %s" % e)
except Exception as e:
# OSError, IOError or whatever envoy will raise
logger.critical("something unexpected happened: %s" % e)
def stopProcess(self, name):
"""Stops a process by name."""
try:
logger.info("Supervisor: stopping %s" % name)
return self.server.supervisor.stopProcess(name)
except xmlrpc.client.Fault as e:
if e.faultCode == 70: # NOT_RUNNING
return True
logger.error("Supervisor: failed to stop %s, fault code: %d" %
(name, e.faultCode))
return False
except IOError:
logger.error("Supervisor: stop %s failed due to ioerror" % name)
return False
def upgrade_endaga(channel):
"""Upgrades the endaga metapackage."""
# Validate.
if channel not in ('stable', 'beta'):
logger.error('cannot upgrade to the "%s" channel' % channel)
return
logger.notice('upgrading the endaga metapackage with channel %s' % channel)
# Update packages.
response = delegator.run('sudo apt-get update')
if response.return_code != 0:
message = 'Error while running "apt-get update": %s' % response.out
logger.error(message)
# Try a dry-run of the upgrade.
command = ('sudo apt-get install --assume-yes --dry-run'
' --only-upgrade -t %s endaga' % channel)
response = delegator.run(command)
if response.return_code != 0:
message = ('Error while dry running the endaga upgrade: %s' %
response.out)
logger.error(message)
return
# Upgrade just the metapackage.
command = ('sudo apt-get install --assume-yes'
' --only-upgrade -t %s endaga' % channel)
response = delegator.run(command)
if response.return_code != 0:
message = 'Error while upgrading endaga: %s' % response.out
logger.error(message)
def ack(self, seqno):
"""Process an ack to the db.
An ack up to a seqno means that all events up to and including that
seqno have been handled and can be safely removed.
"""
try:
cur = self.conn.cursor()
cur.execute("DELETE FROM endaga_events WHERE seqno<=%s;", (seqno,))
# If this fails, we don't commit.
logger.info("EventStore: ack'd seqno %d" % int(seqno))
self.conn.commit()
except BaseException as e:
logger.error("EventStore: ack seqno %s exception %s" % (seqno, e))
raise
def registration_worker(self, from_name, ip, port, ret_num):
try:
# Postcondition: number must be globally registered and set up.
number = self.ic.register_subscriber(imsi=from_name)['number']
subscriber.create_subscriber(from_name, number, ip, port)
self.fs_ic.send_to_number(
number, ret_num,
gt("Your number is %(number)s.") % {'number': number})
reason = 'Provisioned user %s number %s' % (from_name, number)
events.create_provision_event(from_name, reason)
except Exception as e:
self.fs_ic.send_to_imsi(from_name, ip, port, ret_num,
gt("Failed to register your handset."))
logger.error("Failed to provision %s: %s" %
(from_name, traceback.format_exc(e)))
def startProcess(self, name):
"""Starts a process by name."""
try:
logger.info("Supervisor: starting %s" % name)
return self.server.supervisor.startProcess(name)
except xmlrpc.client.Fault as e:
if e.faultCode == 60: # ALREADY_STARTED
logger.info("Supervisor: %s already started" % name)
return True
logger.error("Supervisor: failed to start %s, fault code: %d" %
(name, e.faultCode))
return False
except IOError:
logger.error("Supervisor: start %s failed due to ioerror" % name)
return False
def get_vpn_conf(eapi, csr):
data = {'bts_uuid': snowflake.snowflake(), 'csr': csr}
registration = conf['registry'] + '/bts/register'
try:
r = requests.post(registration, data=data, headers=eapi.auth_header)
if r.status_code == 200:
return json.loads(r.text)
else:
err = ("VPN conf/cert signing failed with status %d (%s)" %
(r.status_code, r.text))
except socket.error as ex:
err = ("socket error connecting to %s: %s" % (registration, ex))
except requests.exceptions.RequestException as ex:
err = ("request to %s failed: %s" % (registration, ex))
logger.error(err)
raise ValueError(err)
def process(processors):
try:
if processors:
p, processors = processors[0], processors[1:]
return p(lambda: process(processors))
else:
return self.handle()
except web.HTTPError as e:
logger.error("Web error: %s" % e)
raise
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
logger.critical("Unhandled exception raised",
traceback=traceback.format_exc())
raise self.internalerror()
def process(self, resp_dict):
"""Process sections of a checkin response.
Right now we have three sections: config, events, and subscribers.
"""
if 'status' in resp_dict and resp_dict['status'] == 'deregistered':
reset_registration()
for section in resp_dict:
if section == CheckinHandler.CONFIG_SECTION:
self.process_config(resp_dict[section])
elif section == CheckinHandler.EVENTS_SECTION:
self.process_events(resp_dict[section])
elif section == CheckinHandler.SUBSCRIBERS_SECTION:
self.process_subscribers(resp_dict[section])
elif section != 'status':
logger.error("Unexpected checkin section: %s" % section)
def bill(self, to_number, from_number):
try:
if from_number == DASHBOARD_FROM_NUMBER:
self.tariff_type = 'free_sms'
tariff = billing.get_sms_cost(self.tariff_type,
destination_number=to_number)
username = subscriber.get_imsi_from_number(to_number)
if username:
reason = 'SMS from %s to %s (incoming_sms)' % (
from_number, username)
old_balance = subscriber.get_account_balance(username)
subscriber.subtract_credit(username, str(int(tariff)))
events.create_sms_event(username, old_balance, tariff, reason,
to_number, from_number=from_number)
except Exception as e:
logger.error("Endaga bill error:" + traceback.format_exc(e))
def _runCommand(self, name, command):
if not self.cmd:
try:
r = delegator.run("systemctl --version")
if r.return_code == 0:
self.cmd = "systemctl"
else:
self.cmd = "service"
except BaseException as e:
logger.error("delegator systemctl exception %s" % e)
self.cmd = "service"
if self.cmd == "systemctl":
r = delegator.run("sudo systemctl %s %s" % (command, name))
else:
r = delegator.run("sudo service %s %s" % (name, command))
result = r.return_code == 0
return result
def _runCommand(self, name, command):
if not self.cmd:
try: # envoy.run may throw due to an internal bug in timeout handling
r = envoy.run("systemctl --version", timeout=2)
if r.status_code == 0:
self.cmd = "systemctl"
else:
self.cmd = "service"
except BaseException as e:
logger.error("envoy systemctl exception %s" % e)
self.cmd = "service"
if self.cmd == "systemctl":
r = envoy.run("sudo systemctl %s %s" % (command, name))
else:
r = envoy.run("sudo service %s %s" % (name, command))
result = r.status_code == 0
return result
def update_vpn():
"""
If the BTS is registered, try to start the VPN. If the BTS is not
registered, skip.
If the BTS is unregistered (on the dashboard), no services are available.
Regardless of whether the VPN is up or down, all services should be started
(this will enable disconnected operation). However, when the VPN comes
online, we need to restart FS to make sure that we're bound to the VPN IP
so outgoing calls can work.
"""
if not ('bts_registered' in conf and conf['bts_registered']):
logger.error('BTS is not yet registered, skipping VPN setup, killing'
' all services.')
for s in SERVICES:
if s.name == 'endagad':
continue
s.stop()
return
# If the VPN is down, try to start it, then restart FS if we succeed.
if not system_utilities.get_vpn_ip():
max_attempts = 10
for _ in range(0, max_attempts):
# Sometimes the vpn service is started, but the VPN is still down.
# If this is the case, stop the vpn service first.
openvpn_service = Service.SystemService('ccm-openvpn')
if openvpn_service.status() == ServiceState.Running:
openvpn_service.stop()
if openvpn_service.start():
logger.notice('VPN service started')
if system_utilities.get_vpn_ip():
logger.notice('VPN up - restarting freeswitch')
Service.SystemService('freeswitch').restart()
else:
logger.error('VPN interface (%s) is down' %
conf.get('external_interface'))
else:
logger.error(
'VPN failed to start after registration, retrying.')
time.sleep(3)
if not system_utilities.get_vpn_ip():
logger.error('Failed to set up VPN after %d attempts!' %
max_attempts)
# Start all the other services. This is safe to run if services are
# already started.
for s in SERVICES:
try:
s.start()
except Exception as e:
logger.critical("Exception %s while starting %s" % (e, s.name))
def set_seqno(self, seqno):
"""Sets the current event seqno to the given value.
This becomes the CURRENT seqno of the table; the next seqno generated
will be this value+1.
We have to grab a *full table lock* on this, otherwise we can have
issues with writes duplicating the seqno. We should only rarely call
this -- the only reason the seqno needs to be updated is if we're
restoring a DB or cloning a BTS.
"""
cur = self.conn.cursor()
try:
cur.execute("LOCK TABLE endaga_events IN EXCLUSIVE MODE;")
cur.execute("SELECT setval('endaga_events_seqno_seq', %s);",
(seqno,))
self.conn.commit()
except BaseException:
logger.error("EventStore: set seqno %s failed" % seqno)
self.conn.rollback()
def get_service_tariff(service_type, activity_type, destination_number=''):
"""Get the tariff for the given service type.
Prices are stored in the ConfigDB and are of the form
'prices.on_network_send.cost_to_subscriber_per_min' or, if it's an outbound
key, the prefix is included:
'prices.off_network_send.56.cost_to_subscriber_per_sms'
Args:
service_type: one of off_network_send, off_network_receive,
on_network_send, off_network_receive, free or error
activity_type: call or sms
destination_number: the number we're calling or texting
Returns:
integer value of service tariff if the type exists
None if no such type.
"""
# Certain service types are free.
if 'free' in service_type or 'error' in service_type:
return 0
service_type = convert_legacy_service_type(service_type)
# Set the cost key suffix.
if activity_type == 'call':
cost_key = 'cost_to_subscriber_per_min'
elif activity_type == 'sms':
cost_key = 'cost_to_subscriber_per_sms'
# Lookup the prefix if a destination number is set.
if destination_number and service_type == 'off_network_send':
prefix = get_prefix_from_number(destination_number)
key = 'prices.%s.%s.%s' % (service_type, prefix, cost_key)
else:
key = 'prices.%s.%s' % (service_type, cost_key)
# Finally lookup the actual cost.
try:
return int(config_db[key])
except KeyError:
logger.error("get_service_tariff lookup failed for key: %s" % key)
return 0
def get_service_billable_unit(service_type, destination_number):
""" Gets the billable unit for a service type and destination. Default is 1
second (this is used when we don't have a matching billable unit).
"""
# Certain service types are free.
if 'free' in service_type or 'error' in service_type:
return 1
service_type = convert_legacy_service_type(service_type)
prefix = get_prefix_from_number(destination_number)
if service_type == "off_network_send":
key = 'prices.%s.%s.billable_unit' % (service_type, prefix)
else:
key = 'prices.%s.billable_unit' % (service_type)
try:
return int(config_db[key])
except KeyError:
logger.error("get_service_billable_unit lookup failed for key: %s" % key)
return 1
def sms_worker(self, to, from_num, from_name, body, service_type):
"""The SMS worker that runs in its own thread.
Fails with a logger entry if we fail to post to the billing endpoint.
Args:
message params
"""
try:
# TODO(matt): handle else (message failed to send).
if self.interconnect_client.send(to, from_num, body):
billing_url = self.conf['billing_url']
params = {
"from_name": from_name,
"from_number": from_num,
"destination": to,
"service_type": service_type
}
requests.post(billing_url, data=params)
except Exception as e:
logger.error("Endaga " + traceback.format_exc(e))
def check_signed_params(self, jwt_data):
"""
Decodes the params, makes sure they pass signature (i.e., are valid),
and then checks that we haven't seen the msgid before. Raises a
ValueError if errors, else returns True.
TODO(matt): this particular method seems to be unused (not so the one
in federer_handlers.config.config).
"""
s = itsdangerous.JSONWebSignatureSerializer(self.conf['bts_secret'])
try:
data = s.loads(jwt_data)
except itsdangerous.BadSignature:
logger.error("Bad jwt signature for request, ignoring.")
raise ValueError("Bad signature")
# make sure the msg hasn't been seen before, if so, discard it
if "msgid" in data:
if self.msgid_db.seen(str(data['msgid'])):
logger.error("Endaga: Repeat msgid: %s" % (data['msgid'],))
raise ValueError("Repeat msgid: %s" % (data['msgid'],))
else:
logger.error("Endaga: No message ID.")
raise ValueError("No message ID.")
return data
def register_update(eapi):
"""Ensures the inbound URL for the BTS is up to date."""
vpn_ip = system_utilities.get_vpn_ip()
vpn_status = 'up' if vpn_ip else 'down'
# This could fail when offline! Must handle connection exceptions.
params = {
'bts_uuid': _get_snowflake(),
'vpn_status': vpn_status,
'vpn_ip': vpn_ip,
'federer_port': '80',
}
try:
d = _send_cloud_req(requests.get,
'/bts/register',
'BTS registration',
params=params,
headers=eapi.auth_header,
timeout=11)
if 'bts_secret' in d:
conf['bts_secret'] = d['bts_secret']
except RegistrationError as ex:
logger.error(str(ex))
def update_vpn():
"""
If the BTS is registered, try to start the VPN. If the BTS is not
registered, skip.
If the BTS is unregistered (on the dashboard), no services are available.
Regardless of whether the VPN is up or down, all services should be started
(this will enable disconnected operation). However, when the VPN comes
online, we need to restart FS to make sure that we're bound to the VPN IP
so outgoing calls can work.
"""
if not ('bts_registered' in conf and conf['bts_registered']):
logger.error("BTS is not yet registered, skipping VPN setup, killing"
" all services.")
for s in SERVICES:
if s.name == "endagad":
continue
s.stop()
return
# If the VPN is down, try to start it, then restart FS if we succeed.
if not system_utilities.get_vpn_ip():
max_attempts = 10
for _ in range(0, max_attempts):
# Sometimes the vpn service is started, but the VPN is still down.
# If this is the case, stop the vpn service first.
openvpn_service = Service.SupervisorService("openvpn")
if openvpn_service.status() == ServiceState.Running:
openvpn_service.stop()
if (openvpn_service.start() and system_utilities.get_vpn_ip()):
logger.notice("VPN up restarting services")
Service.SystemService("freeswitch").restart()
else:
logger.error("VPN didn't come up after registration,"
" retrying.")
time.sleep(3)
if not system_utilities.get_vpn_ip():
logger.error("Failed to set up VPN after %d attempts!" %
max_attempts)
# Start all the other services. This is safe to run if services are
# already started.
for s in SERVICES:
s.start()
def run(self):
"""
Main loop for endagad. This moves the system through the various
states of operation -- it should be a state machine really!
General flow is:
1) Tries to get configuration from server to produce VPN keys
2) Generates keys locally.
3) Sends CSR for signing, returns that.
4) Starts system services (FS, BTS, etc) and configures them
appropriately. Note configuration can change depending on registration
and VPN state of the system.
5) Runs checkin periodically.
"""
eapi = interconnect.endaga_ic(self._conf)
if 'registration_interval' not in self._conf:
self._conf['registration_interval'] = 60
UNHEALTHY_THRESH = self._conf.get('bts.unhealthy_threshold', 3)
unhealthy_count = UNHEALTHY_THRESH # fail quickly on first pass
while True:
# Retrieve keys/tokens, or do nothing if we have them.
logger.notice("Performing gen_keys")
registration.generate_keys()
# generate_keys() loads auth token on success. Need to update the
# interconnect client's token if so.
if eapi.token is None:
eapi.token = self._conf['endaga_token']
# Try to register/get VPN credentials. Tries forever if fails.
logger.notice("Performing register")
registration.register(eapi)
# Registered, start services and tries to start VPN. Stop
# everything otherwise.
logger.notice("Performing clear_pid")
registration.clear_old_pid()
logger.notice("Performing update_vpn")
registration.update_vpn()
# At this point, all services should be up, so we can perform
# additional configuration.
self._reset_bts_config()
# Update the inbound_url if the VPN is up.
if system_utilities.get_vpn_ip() is not None:
logger.notice("Performing register_update")
registration.register_update(eapi)
logger.notice("Performing ensure_fs_external_bound")
registration.ensure_fs_external_bound_to_vpn()
# Send checkin to cloud
try:
# Sends events, tries to get config info. Can proceed w/o VPN.
logger.notice("Performing checkin.")
checkin_data = eapi.checkin(timeout=30)
logger.notice("Performing system health check.")
if not registration.system_healthcheck(checkin_data):
unhealthy_count += 1
logger.notice("System unhealthy: %d" % unhealthy_count)
else:
unhealthy_count = 0
except (ConnectionError, Timeout):
logger.error(
"checkin failed due to connection error or timeout.")
except BSSError as e:
logger.error("bts exception: %s" % e)
if unhealthy_count > UNHEALTHY_THRESH:
logger.notice("BTS seems unhealthy, restarting BTS services.")
bts.restart()
Service.SystemService("freeswitch").restart()
# Upgrade the endaga metapackage, when appropriate and only if that
# feature is enabled.
logger.notice("Performing autoupgrade")
system_utilities.try_to_autoupgrade()
# Sleep for some amount of time before retrying
logger.notice("Performing sleep")
time.sleep(self._conf['registration_interval'])
def checkin(self, timeout=11):
"""Gather system status."""
# Compile checkin data
checkin_start = time.time()
status = {
'usage': events.usage(),
'uptime': system_utilities.uptime(),
'system_utilization': self.utilization_tracker.get_data(),
}
# Append status if we can
try:
#get the software versions
status['versions'] = bts.get_versions()
except BSSError as e:
logger.error("bts get_versions error: %s" % e)
try:
# Gather camped subscriber list
status['camped_subscribers'] = bts.active_subscribers()
except BSSError as e:
logger.error("bts get active_subscribers error: %s" % e)
# Gather tower load and noise data.
# NOTE(matt): these values can vary quite a bit over a minute. It
# might be worth capturing data more frequently and sending
# something like average or median values.
status['openbts_load'] = {}
try:
status['openbts_load'] = bts.get_load()
except BSSError as e:
logger.error("bts get_load error: %s" % e)
for key, val in list(self._checkin_load_stats.items()):
status['openbts_load']['checkin.' + key] = val
self._checkin_load_stats.clear()
try:
status['openbts_noise'] = bts.get_noise()
except BSSError as e:
logger.error("bts get_noise error: %s" % e)
status['radio'] = {}
try:
status['radio']['band'] = bts.get_band()
# eventually need to also grab all used channels, not just c0
# TODO: (kheimerl) T13270338 Add multiband support
status['radio']['c0'] = bts.get_arfcn_c0()
#also add power here eventually
# TODO: (kheimerl) T13270365 Add power level support
except BSSError as e:
#delete the key if this failed
del status['radio']
logger.error("bts radio error: %s" % e)
# Add balance sync data
status['subscribers'] = subscriber.get_subscriber_states(
imsis=events.EventStore().modified_subs())
# Add delta protocol context (if available) to let server know,
# client supports delta optimization & has a prior delta state
if delta.DeltaProtocol.CTX_KEY not in status: # just a precaution
sections_ctx = {}
for section, ctx in list(CheckinHandler.section_ctx.items()):
if ctx:
sections_ctx[section] = ctx.to_proto_dict()
if sections_ctx:
status[delta.DeltaProtocol.CTX_KEY] = {
delta.DeltaProtocolOptimizer.SECTIONS_CTX_KEY: sections_ctx
}
# Send checkin request.
uuid = snowflake.snowflake()
data = {
'status': status,
'bts_uuid': uuid,
}
headers = dict(self.auth_header)
# Set content type to app/json & utf-8, compressed or not - JSON should
# be more efficient then URL encoded JSON form payload
headers['Content-Type'] = 'application/json; charset=utf-8'
data_json = json.dumps(data)
decompressed_status_len = len(data_json)
status_len = decompressed_status_len
if status_len > endaga_ic.MIN_COMPRESSIBLE_REQUEST_SZ:
# try to gzip payload, send uncompressed if compression failed
try:
gzbuf = BytesIO()
with GzipFile(mode='wb', fileobj=gzbuf) as gzfile:
gzfile.write(bytes(data_json, encoding='UTF-8'))
data_json = gzbuf.getvalue()
# Using Content-Encoding header since AWS cannot handle
# Transfer-Encoding header which would be more appropriate here
headers['Content-Encoding'] = 'gzip'
status_len = len(data_json) # set len to reflect compression
except BaseException as e:
logger.error("Checkin request Gzip error: %s" % e)
headers['Content-Length'] = str(status_len)
post_start = time.time()
try:
r = self.session.post(
self.conf['registry'] + "/checkin?id=" +
# add part of uuid to the query, it helps with
# debugging & server side logging and can
# be used by LBs
uuid[:8],
headers=headers,
data=data_json,
timeout=timeout,
cookies=self._session_cookies)
except BaseException as e:
logger.error("Endaga: checkin failed , network error: %s." % e)
self._cleanup_session()
self._checkin_load_stats['req_sz'] = status_len
self._checkin_load_stats['raw_req_sz'] = decompressed_status_len
self._checkin_load_stats['post_lat'] = time.time() - post_start
raise
post_end = time.time()
# Make sure either server sent charset or we set it to utf-8 (JSON
# default)
if not r.encoding:
r.encoding = 'utf-8'
text = r.text
decompressed_response_len = len(text)
response_len = decompressed_response_len
# Try to get correct content length from HTTP headers, it should
# reflect correctly compressed length. if it fails - fall back to
# getting length of returned text
cont_len = r.headers.get('Content-Length')
if cont_len:
try:
response_len = int(cont_len)
except BaseException:
pass
if r.status_code == 200:
try:
CheckinHandler(text)
logger.info("Endaga: checkin success.")
if r.cookies is not None:
if self._session_cookies is None:
# First time cookies are seen from server
# initialize the cookies dict
self._session_cookies = dict(r.cookies)
else:
for key, value in r.cookies.items():
# if server sent new/updated cookies, update them,
# but keep previously set cokies as well. ELBs
# do not send AWSELB cookies on every request &
# expect clients to 'remember' them
self._session_cookies[key] = value
except BaseException:
self._cleanup_session()
raise
else:
logger.error("Endaga: checkin failed (%d), reason: %s, body: %s" %
(r.status_code, r.reason, r.text))
# cleanup session on any error
if r.status_code >= 300:
self._cleanup_session()
checkin_end = time.time()
self._checkin_load_stats['req_sz'] = status_len # request payload SZ
self._checkin_load_stats['raw_req_sz'] = decompressed_status_len
self._checkin_load_stats[
'rsp_sz'] = response_len # response payload SZ
self._checkin_load_stats['raw_rsp_sz'] = decompressed_response_len
# Checkin Latencies
self._checkin_load_stats['post_lat'] = post_end - post_start
self._checkin_load_stats['process_lat'] = checkin_end - post_end
self._checkin_load_stats['lat'] = checkin_end - checkin_start
data['response'] = {'status': r.status_code, 'text': r.text}
return data
