def process_bts_settings(self, data_dict):
"""Process bts settings.
TODO: We should revisit how configs are stored on cloud
"""
settings_map = {
'GSM.Identity.MCC':
{'get': self.get_mcc,
'set': self.set_mcc},
'GSM.Identity.MNC':
{'get': self.get_mnc,
'set': self.set_mnc},
'GSM.Identity.ShortName':
{'get': self.get_short_name,
'set': self.set_short_name},
'Control.LUR.OpenRegistration':
{'get': self.get_open_registration,
'set': self.set_open_registration},
'GSM.Radio.C0':
{'get': self.get_arfcn_c0,
'set': self.set_arfcn_c0},
'GSM.Timer.T3212':
{'get': lambda: self.get_timer('3212'),
'set': lambda minutes: self.set_timer('3212', minutes)},
}
for (key, val) in data_dict.items():
try:
cur_val = settings_map[key]['get']()
if cur_val != val:
settings_map[key]['set'](val)
logger.info("Changed bts setting: %s = %s (was %s)" %
(key, val, cur_val))
except Exception as e:
logger.error("Failed to process openbts setting"
"(%s): %s = %s" % (e, key, val))
def log_worker(self, msgid, window_start, window_end, log_name):
logger.info("Log req %s started" % msgid)
log_path = "/var/log/%s" % log_name
tmp_file = ('/tmp/%s-%s.log.gz' % (log_name, msgid))
with gzip.open(tmp_file, 'w') as f:
for msg in log_stream(log_path, window_start, window_end):
f.write(msg)
params = {'msgid': msgid, 'log_name': log_name}
files = {'file': open(tmp_file, 'rb')}
r = requests.post(self.conf['registry'] + "/bts/logfile",
data=params,
files=files,
headers=self.ic.auth_header)
try:
if r.status_code == 200:
logger.info("Log req %s posted successfully" % msgid)
else:
logger.error("Log req %s responded with %s: %s" %
(msgid, r.status_code, r.text))
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as e:
logger.error("Log req %s failed: %s" % (msgid, e))
os.unlink(tmp_file)
def register(eapi):
"""
After a call to this, BTS should be registered with the server and have all
credentials to connect to VPN.
"Registered" here means that we have gotten our CSR (created by
generate_keys()) signed by the server, and have received a signed
certificate and client configuration back from the server. We'll stay stuck
here trying to get our key signed until we succeed.
A device is associated with an account in get_registration_conf(); if this
fails, then the device won't have access credentials for our API and none
of this will succeed. A device that is associated with an account, but for
some reason can't get VPN credentials, will get stuck here, but local
services will still work.
"""
if not ('bts_registered' in conf and conf['bts_registered']):
# We're not registered yet, so do the initial registration procedure.
OPENVPN_DIR = '/etc/openvpn/endaga-'
CLIENT_CERT = OPENVPN_DIR + 'client.crt'
VPN_CONF = OPENVPN_DIR + 'vpn-client.conf.noauto'
# If we don't have a client certificate (signed by the certifier)
# and VPN config file then attempt to get them by registering.
if not (os.path.exists(CLIENT_CERT) and os.path.exists(VPN_CONF)):
# Send the CSR and keep trying to register.
with open(OPENVPN_DIR + 'client.req') as f:
csr = f.read()
vpn = _retry_req(lambda: get_vpn_conf(eapi, csr),
'BTS registration')
cert = vpn['certificate']
vpnconf = vpn['vpnconf']
assert len(vpnconf) > 0 and len(cert) > 0, 'Invalid VPN parameters'
logger.info('got VPN configuration')
# write the client cert (before verification, for troubleshooting)
with open(CLIENT_CERT, 'w') as f:
f.write(cert)
# write the VPN config (don't discard if cert verification fails)
with open(VPN_CONF, 'w') as f:
f.write(vpnconf)
else:
# We already have a cert and VPN conf from a previous
# attempt to register, but the cert could not be validated
# against the CA bundle and that attempt aborted. The
# user should be able to replace the CA bundle
# (etage-bundle.crt) with the one that corresponds to the
# cloud installation, after which registration should
# succeed.
with open(CLIENT_CERT, 'r') as f:
cert = f.read()
# validate client cert against CA
system_utilities.verify_cert(
cert,
os.path.dirname(VPN_CONF) + '/etage-bundle.crt')
conf['bts_registered'] = True
def POST(self):
data = web.input()
# The auth token should be here.
logger.info('GET mock subscriber provision')
auth_token = web.ctx.env['HTTP_AUTHORIZATION']
assert data.bts_uuid == snowflake.snowflake()
assert len(data.imsi) == 19
assert auth_token == "Token %s" % CONF['endaga_token']
return json.dumps({ 'number': '1%010d' % random.randint(1000000000, 9999999999) })
def stopProcess(self, name):
"""Stops a process by name."""
try:
logger.info("Supervisor: stopping %s" % name)
return self.server.supervisor.stopProcess(name)
except xmlrpc.client.Fault as e:
if e.faultCode == 70: # NOT_RUNNING
return True
logger.error("Supervisor: failed to stop %s, fault code: %d" %
(name, e.faultCode))
return False
except IOError:
logger.error("Supervisor: stop %s failed due to ioerror" % name)
return False
def startProcess(self, name):
"""Starts a process by name."""
try:
logger.info("Supervisor: starting %s" % name)
return self.server.supervisor.startProcess(name)
except xmlrpc.client.Fault as e:
if e.faultCode == 60: # ALREADY_STARTED
logger.info("Supervisor: %s already started" % name)
return True
logger.error("Supervisor: failed to start %s, fault code: %d" %
(name, e.faultCode))
return False
except IOError:
logger.error("Supervisor: start %s failed due to ioerror" % name)
return False
def ack(self, seqno):
"""Process an ack to the db.
An ack up to a seqno means that all events up to and including that
seqno have been handled and can be safely removed.
"""
try:
cur = self.conn.cursor()
cur.execute("DELETE FROM endaga_events WHERE seqno<=%s;", (seqno,))
# If this fails, we don't commit.
logger.info("EventStore: ack'd seqno %d" % int(seqno))
self.conn.commit()
except BaseException as e:
logger.error("EventStore: ack seqno %s exception %s" % (seqno, e))
raise
def verify_cert(_, ca_file):
""" Validate that cert has been signed by the Etage CA. """
r = delegator.run("openssl verify -CAfile %s %s/endaga-client.crt" %
(ca_file, os.path.dirname(ca_file)))
if r.return_code != 0:
"""
Any error requires manual intervention, i.e., updating the CA
cert, and hence cannot be resolved by retrying
registration. Therefore we just raise an exception that
terminates the agent.
"""
msg = ("Unable to verify client cert against CA bundle:\n%s" %
(r.out))
logger.critical(msg)
raise SystemExit(msg)
logger.info("Verified client cert against CA %s" % (ca_file, ))
def register(eapi):
"""
After a call to this, BTS should be registered with the server and have all
credentials to connect to VPN.
"Registered" here means that we have gotten our CSR (created by
generate_keys()) signed by the server, and have received a signed
certificate and client configuration back from the server. We'll stay stuck
here trying to get our key signed until we succeed.
A device is associated with an account in get_registration_conf(); if this
fails, then the device won't have access credentials for our API and none
of this will succeed. A device that is associated with an account, but for
some reason can't get VPN credentials, will get stuck here, but local
services will still work.
"""
if not ('bts_registered' in conf and conf['bts_registered']):
# We're not registered yet, so do the initial registration procedure.
# Send on the CSR and keep trying to register.
backoff_count = 0
with open("/etc/openvpn/endaga-client.req") as f:
csr = f.read()
while not ('bts_registered' in conf and conf['bts_registered']):
try:
vpn = get_vpn_conf(eapi, csr)
cert = vpn['certificate']
vpnconf = vpn['vpnconf']
assert len(vpnconf) > 0 and len(cert) > 0, 'Empty VPN Response'
with open('/etc/openvpn/endaga-client.crt', 'w') as f:
f.write(cert)
with open('/etc/openvpn/endaga-vpn-client.conf.noauto',
'w') as f:
f.write(vpnconf)
conf['bts_registered'] = True
except ValueError:
delay = min(2**backoff_count, 300)
logger.info("registration failed, sleeping %d seconds" %
(delay, ))
time.sleep(delay)
backoff_count += 1
def checkin(self, timeout=11):
"""Gather system status."""
# Compile checkin data
checkin_start = time.time()
status = {
'usage': events.usage(),
'uptime': system_utilities.uptime(),
'system_utilization': self.utilization_tracker.get_data(),
}
# Append status if we can
try:
#get the software versions
status['versions'] = bts.get_versions()
except BSSError as e:
logger.error("bts get_versions error: %s" % e)
try:
# Gather camped subscriber list
status['camped_subscribers'] = bts.active_subscribers()
except BSSError as e:
logger.error("bts get active_subscribers error: %s" % e)
# Gather tower load and noise data.
# NOTE(matt): these values can vary quite a bit over a minute. It
# might be worth capturing data more frequently and sending
# something like average or median values.
status['openbts_load'] = {}
try:
status['openbts_load'] = bts.get_load()
except BSSError as e:
logger.error("bts get_load error: %s" % e)
for key, val in list(self._checkin_load_stats.items()):
status['openbts_load']['checkin.' + key] = val
self._checkin_load_stats.clear()
try:
status['openbts_noise'] = bts.get_noise()
except BSSError as e:
logger.error("bts get_noise error: %s" % e)
status['radio'] = {}
try:
status['radio']['band'] = bts.get_band()
# eventually need to also grab all used channels, not just c0
# TODO: (kheimerl) T13270338 Add multiband support
status['radio']['c0'] = bts.get_arfcn_c0()
#also add power here eventually
# TODO: (kheimerl) T13270365 Add power level support
except BSSError as e:
#delete the key if this failed
del status['radio']
logger.error("bts radio error: %s" % e)
# Add balance sync data
status['subscribers'] = subscriber.get_subscriber_states(
imsis=events.EventStore().modified_subs())
# Add delta protocol context (if available) to let server know,
# client supports delta optimization & has a prior delta state
if delta.DeltaProtocol.CTX_KEY not in status: # just a precaution
sections_ctx = {}
for section, ctx in list(CheckinHandler.section_ctx.items()):
if ctx:
sections_ctx[section] = ctx.to_proto_dict()
if sections_ctx:
status[delta.DeltaProtocol.CTX_KEY] = {
delta.DeltaProtocolOptimizer.SECTIONS_CTX_KEY: sections_ctx
}
# Send checkin request.
uuid = snowflake.snowflake()
data = {
'status': status,
'bts_uuid': uuid,
}
headers = dict(self.auth_header)
# Set content type to app/json & utf-8, compressed or not - JSON should
# be more efficient then URL encoded JSON form payload
headers['Content-Type'] = 'application/json; charset=utf-8'
data_json = json.dumps(data)
decompressed_status_len = len(data_json)
status_len = decompressed_status_len
if status_len > endaga_ic.MIN_COMPRESSIBLE_REQUEST_SZ:
# try to gzip payload, send uncompressed if compression failed
try:
gzbuf = BytesIO()
with GzipFile(mode='wb', fileobj=gzbuf) as gzfile:
gzfile.write(bytes(data_json, encoding='UTF-8'))
data_json = gzbuf.getvalue()
# Using Content-Encoding header since AWS cannot handle
# Transfer-Encoding header which would be more appropriate here
headers['Content-Encoding'] = 'gzip'
status_len = len(data_json) # set len to reflect compression
except BaseException as e:
logger.error("Checkin request Gzip error: %s" % e)
headers['Content-Length'] = str(status_len)
post_start = time.time()
try:
r = self.session.post(
self.conf['registry'] + "/checkin?id=" +
# add part of uuid to the query, it helps with
# debugging & server side logging and can
# be used by LBs
uuid[:8],
headers=headers,
data=data_json,
timeout=timeout,
cookies=self._session_cookies)
except BaseException as e:
logger.error("Endaga: checkin failed , network error: %s." % e)
self._cleanup_session()
self._checkin_load_stats['req_sz'] = status_len
self._checkin_load_stats['raw_req_sz'] = decompressed_status_len
self._checkin_load_stats['post_lat'] = time.time() - post_start
raise
post_end = time.time()
# Make sure either server sent charset or we set it to utf-8 (JSON
# default)
if not r.encoding:
r.encoding = 'utf-8'
text = r.text
decompressed_response_len = len(text)
response_len = decompressed_response_len
# Try to get correct content length from HTTP headers, it should
# reflect correctly compressed length. if it fails - fall back to
# getting length of returned text
cont_len = r.headers.get('Content-Length')
if cont_len:
try:
response_len = int(cont_len)
except BaseException:
pass
if r.status_code == 200:
try:
CheckinHandler(text)
logger.info("Endaga: checkin success.")
if r.cookies is not None:
if self._session_cookies is None:
# First time cookies are seen from server
# initialize the cookies dict
self._session_cookies = dict(r.cookies)
else:
for key, value in r.cookies.items():
# if server sent new/updated cookies, update them,
# but keep previously set cokies as well. ELBs
# do not send AWSELB cookies on every request &
# expect clients to 'remember' them
self._session_cookies[key] = value
except BaseException:
self._cleanup_session()
raise
else:
logger.error("Endaga: checkin failed (%d), reason: %s, body: %s" %
(r.status_code, r.reason, r.text))
# cleanup session on any error
if r.status_code >= 300:
self._cleanup_session()
checkin_end = time.time()
self._checkin_load_stats['req_sz'] = status_len # request payload SZ
self._checkin_load_stats['raw_req_sz'] = decompressed_status_len
self._checkin_load_stats[
'rsp_sz'] = response_len # response payload SZ
self._checkin_load_stats['raw_rsp_sz'] = decompressed_response_len
# Checkin Latencies
self._checkin_load_stats['post_lat'] = post_end - post_start
self._checkin_load_stats['process_lat'] = checkin_end - post_end
self._checkin_load_stats['lat'] = checkin_end - checkin_start
data['response'] = {'status': r.status_code, 'text': r.text}
return data
def post(self, request, format=None):
"""Handle POST requests."""
# First make sure the dom parses.
try:
dom = xml.dom.minidom.parseString(request.POST['cdr'])
except xml.parsers.expat.ExpatError:
logger.warning("invalid XML CDR: '%s'" % (request.POST['cdr'], ))
return Response("Bad XML", status=status.HTTP_400_BAD_REQUEST)
except KeyError:
logger.warning("invalid POST request: '%s'" % (request.POST, ))
return Response("Missing CDR", status=status.HTTP_400_BAD_REQUEST)
# Then make sure all of the necessary pieces are there. Fail if any
# required tags are missing
data = {}
for tag_name in [
"billsec", "username", "caller_id_name", "network_addr",
"destination_number"
]:
data[tag_name] = dom.getElementsByTagName(tag_name)
for tag_name in data:
if not data[tag_name]:
return Response("Missing XML",
status=status.HTTP_400_BAD_REQUEST)
else:
data[tag_name] = self.getText(data[tag_name][0].childNodes)
# Convert certain tags to ints.
for tag_name in ["billsec"]:
data[tag_name] = int(data[tag_name])
# Try to get Number instances for both the caller and recipient.
# If we find both Numbers in the system then a user on one Endaga
# network is calling a subscriber on a different Endaga-managed
# network, cool. In our cloud freeswitch instance we actually
# "short circuit" this call so it never goes to Nexmo, but to the
# operators the call is a regular incoming / outgoing event, so we
# will bill it as such.
caller_number, dest_number = None, None
try:
caller_number = Number.objects.get(number=data["caller_id_name"])
try:
caller_cost = caller_number.network.calculate_operator_cost(
'off_network_send',
'call',
destination_number=data['destination_number'])
except ValueError as ex:
# this is raised iff the destination has an invalid prefix
logger.error("invalid number prefix: %s" % (ex, ))
caller_cost = 0
caller_number.network.bill_for_call(caller_cost, data['billsec'],
'outside_call')
logger.info("billed network '%s' for outgoing call: %d" %
(caller_number.network.name, caller_cost))
except Number.DoesNotExist:
pass
try:
dest_number = Number.objects.get(number=data["destination_number"])
# cost to receive doesn't take source/caller into account
dest_cost = dest_number.network.calculate_operator_cost(
'off_network_receive', 'call')
dest_number.network.bill_for_call(dest_cost, data['billsec'],
'incoming_call')
logger.info("billed network '%s' for incoming call: %d" %
(dest_number.network.name, dest_cost))
except Number.DoesNotExist:
pass
if not (caller_number or dest_number):
# We didn't find either Number, that's a failure.
return Response("Invalid caller and destination",
status=status.HTTP_404_NOT_FOUND)
# Local, incoming and outgoing calls respond with 200 OK.
return Response("", status=status.HTTP_200_OK)
def GET(self, uuid):
logger.info('GET mock number allocation with env: %s' % str(web.ctx.env))
auth_token = web.ctx.env['HTTP_AUTHORIZATION']
assert uuid == snowflake.snowflake()
assert auth_token == "Token %s" % CONF['endaga_token']
return '1%010d' % random.randint(1000000000, 9999999999)
def GET(self):
logger.info('GET mock bts register')
return web.forbidden()
def _create_event(imsi,
old_credit,
new_credit,
reason,
kind=None,
call_duration=None,
billsec=None,
from_imsi=None,
from_number=None,
to_imsi=None,
to_number=None,
tariff=None,
up_bytes=None,
down_bytes=None,
timespan=None,
write=True):
"""Logs a generic UsageEvent in the EventStore.
Also writes this action to logger.
Args:
imsi: the IMSI connected to this event
old_credit: the account's balance before this action
new_credit: the account's balance after this action
reason: a string describing this event
kind: the type of event. If None, we will attempt to lookup the type
based on the reason.
call_duration: duration, including connect, if it was a call (seconds)
billsec: billable duration of the event if it was a call (seconds)
from_imsi: sender IMSI
from_number: sender number
to_imsi: destination IMSI
to_number: destination number
tariff: the cost per unit applied during this transaction
up_bytes: integer amount of data uploaded during the timespan
down_bytes: integer amount of data downloaded during the timespan
timsespan: number of seconds over which this measurement was taken
write: write event to the eventstore (default: True; only for tests)
Returns:
A dictionary representing the event
"""
template = ('new event: user: %s, old_credit: %d, new_credit: %d,'
' change: %d, reason: %s\n')
message = template % (imsi, old_credit, new_credit,
new_credit - old_credit, reason)
logger.info(message)
# Add this event to the DB. This is the canonical definition of a
# UsageEvent.
# Version 5: added up_bytes, down_bytes and timespan
# Version 4: added billsec, call_duration (removed underscore)
# Version 3: ~~a mystery~~
# Version 2: added 'call duration', from_imsi, to_imsi, from_number,
# to_number
# Version 1: date, imsi, oldamt, newamt, change, reason, kind
if not kind:
kind = kind_from_reason(reason)
data = {
'date': time.strftime('%Y-%m-%d %H:%M:%S'),
'imsi': imsi,
'oldamt': old_credit,
'newamt': new_credit,
'change': new_credit - old_credit,
'reason': reason,
'kind': kind,
'call_duration': call_duration,
'billsec': billsec,
'from_imsi': from_imsi,
'to_imsi': to_imsi,
'from_number': from_number,
'to_number': to_number,
'tariff': tariff,
'up_bytes': up_bytes,
'down_bytes': down_bytes,
'timespan': timespan,
'version': 5,
}
# TODO(shasan): find a way to remove this and mock out in testing instead
if write:
event_store = EventStore()
event_store.add(data)
return data
