def process_bts_settings(self, data_dict): """Process bts settings. TODO: We should revisit how configs are stored on cloud """ settings_map = { 'GSM.Identity.MCC': {'get': self.get_mcc, 'set': self.set_mcc}, 'GSM.Identity.MNC': {'get': self.get_mnc, 'set': self.set_mnc}, 'GSM.Identity.ShortName': {'get': self.get_short_name, 'set': self.set_short_name}, 'Control.LUR.OpenRegistration': {'get': self.get_open_registration, 'set': self.set_open_registration}, 'GSM.Radio.C0': {'get': self.get_arfcn_c0, 'set': self.set_arfcn_c0}, 'GSM.Timer.T3212': {'get': lambda: self.get_timer('3212'), 'set': lambda minutes: self.set_timer('3212', minutes)}, } for (key, val) in data_dict.items(): try: cur_val = settings_map[key]['get']() if cur_val != val: settings_map[key]['set'](val) logger.info("Changed bts setting: %s = %s (was %s)" % (key, val, cur_val)) except Exception as e: logger.error("Failed to process openbts setting" "(%s): %s = %s" % (e, key, val))
def log_worker(self, msgid, window_start, window_end, log_name): logger.info("Log req %s started" % msgid) log_path = "/var/log/%s" % log_name tmp_file = ('/tmp/%s-%s.log.gz' % (log_name, msgid)) with gzip.open(tmp_file, 'w') as f: for msg in log_stream(log_path, window_start, window_end): f.write(msg) params = {'msgid': msgid, 'log_name': log_name} files = {'file': open(tmp_file, 'rb')} r = requests.post(self.conf['registry'] + "/bts/logfile", data=params, files=files, headers=self.ic.auth_header) try: if r.status_code == 200: logger.info("Log req %s posted successfully" % msgid) else: logger.error("Log req %s responded with %s: %s" % (msgid, r.status_code, r.text)) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: logger.error("Log req %s failed: %s" % (msgid, e)) os.unlink(tmp_file)
def register(eapi): """ After a call to this, BTS should be registered with the server and have all credentials to connect to VPN. "Registered" here means that we have gotten our CSR (created by generate_keys()) signed by the server, and have received a signed certificate and client configuration back from the server. We'll stay stuck here trying to get our key signed until we succeed. A device is associated with an account in get_registration_conf(); if this fails, then the device won't have access credentials for our API and none of this will succeed. A device that is associated with an account, but for some reason can't get VPN credentials, will get stuck here, but local services will still work. """ if not ('bts_registered' in conf and conf['bts_registered']): # We're not registered yet, so do the initial registration procedure. OPENVPN_DIR = '/etc/openvpn/endaga-' CLIENT_CERT = OPENVPN_DIR + 'client.crt' VPN_CONF = OPENVPN_DIR + 'vpn-client.conf.noauto' # If we don't have a client certificate (signed by the certifier) # and VPN config file then attempt to get them by registering. if not (os.path.exists(CLIENT_CERT) and os.path.exists(VPN_CONF)): # Send the CSR and keep trying to register. with open(OPENVPN_DIR + 'client.req') as f: csr = f.read() vpn = _retry_req(lambda: get_vpn_conf(eapi, csr), 'BTS registration') cert = vpn['certificate'] vpnconf = vpn['vpnconf'] assert len(vpnconf) > 0 and len(cert) > 0, 'Invalid VPN parameters' logger.info('got VPN configuration') # write the client cert (before verification, for troubleshooting) with open(CLIENT_CERT, 'w') as f: f.write(cert) # write the VPN config (don't discard if cert verification fails) with open(VPN_CONF, 'w') as f: f.write(vpnconf) else: # We already have a cert and VPN conf from a previous # attempt to register, but the cert could not be validated # against the CA bundle and that attempt aborted. The # user should be able to replace the CA bundle # (etage-bundle.crt) with the one that corresponds to the # cloud installation, after which registration should # succeed. with open(CLIENT_CERT, 'r') as f: cert = f.read() # validate client cert against CA system_utilities.verify_cert( cert, os.path.dirname(VPN_CONF) + '/etage-bundle.crt') conf['bts_registered'] = True
def POST(self): data = web.input() # The auth token should be here. logger.info('GET mock subscriber provision') auth_token = web.ctx.env['HTTP_AUTHORIZATION'] assert data.bts_uuid == snowflake.snowflake() assert len(data.imsi) == 19 assert auth_token == "Token %s" % CONF['endaga_token'] return json.dumps({ 'number': '1%010d' % random.randint(1000000000, 9999999999) })
def stopProcess(self, name): """Stops a process by name.""" try: logger.info("Supervisor: stopping %s" % name) return self.server.supervisor.stopProcess(name) except xmlrpc.client.Fault as e: if e.faultCode == 70: # NOT_RUNNING return True logger.error("Supervisor: failed to stop %s, fault code: %d" % (name, e.faultCode)) return False except IOError: logger.error("Supervisor: stop %s failed due to ioerror" % name) return False
def startProcess(self, name): """Starts a process by name.""" try: logger.info("Supervisor: starting %s" % name) return self.server.supervisor.startProcess(name) except xmlrpc.client.Fault as e: if e.faultCode == 60: # ALREADY_STARTED logger.info("Supervisor: %s already started" % name) return True logger.error("Supervisor: failed to start %s, fault code: %d" % (name, e.faultCode)) return False except IOError: logger.error("Supervisor: start %s failed due to ioerror" % name) return False
def ack(self, seqno): """Process an ack to the db. An ack up to a seqno means that all events up to and including that seqno have been handled and can be safely removed. """ try: cur = self.conn.cursor() cur.execute("DELETE FROM endaga_events WHERE seqno<=%s;", (seqno,)) # If this fails, we don't commit. logger.info("EventStore: ack'd seqno %d" % int(seqno)) self.conn.commit() except BaseException as e: logger.error("EventStore: ack seqno %s exception %s" % (seqno, e)) raise
def verify_cert(_, ca_file): """ Validate that cert has been signed by the Etage CA. """ r = delegator.run("openssl verify -CAfile %s %s/endaga-client.crt" % (ca_file, os.path.dirname(ca_file))) if r.return_code != 0: """ Any error requires manual intervention, i.e., updating the CA cert, and hence cannot be resolved by retrying registration. Therefore we just raise an exception that terminates the agent. """ msg = ("Unable to verify client cert against CA bundle:\n%s" % (r.out)) logger.critical(msg) raise SystemExit(msg) logger.info("Verified client cert against CA %s" % (ca_file, ))
def register(eapi): """ After a call to this, BTS should be registered with the server and have all credentials to connect to VPN. "Registered" here means that we have gotten our CSR (created by generate_keys()) signed by the server, and have received a signed certificate and client configuration back from the server. We'll stay stuck here trying to get our key signed until we succeed. A device is associated with an account in get_registration_conf(); if this fails, then the device won't have access credentials for our API and none of this will succeed. A device that is associated with an account, but for some reason can't get VPN credentials, will get stuck here, but local services will still work. """ if not ('bts_registered' in conf and conf['bts_registered']): # We're not registered yet, so do the initial registration procedure. # Send on the CSR and keep trying to register. backoff_count = 0 with open("/etc/openvpn/endaga-client.req") as f: csr = f.read() while not ('bts_registered' in conf and conf['bts_registered']): try: vpn = get_vpn_conf(eapi, csr) cert = vpn['certificate'] vpnconf = vpn['vpnconf'] assert len(vpnconf) > 0 and len(cert) > 0, 'Empty VPN Response' with open('/etc/openvpn/endaga-client.crt', 'w') as f: f.write(cert) with open('/etc/openvpn/endaga-vpn-client.conf.noauto', 'w') as f: f.write(vpnconf) conf['bts_registered'] = True except ValueError: delay = min(2**backoff_count, 300) logger.info("registration failed, sleeping %d seconds" % (delay, )) time.sleep(delay) backoff_count += 1
def checkin(self, timeout=11): """Gather system status.""" # Compile checkin data checkin_start = time.time() status = { 'usage': events.usage(), 'uptime': system_utilities.uptime(), 'system_utilization': self.utilization_tracker.get_data(), } # Append status if we can try: #get the software versions status['versions'] = bts.get_versions() except BSSError as e: logger.error("bts get_versions error: %s" % e) try: # Gather camped subscriber list status['camped_subscribers'] = bts.active_subscribers() except BSSError as e: logger.error("bts get active_subscribers error: %s" % e) # Gather tower load and noise data. # NOTE(matt): these values can vary quite a bit over a minute. It # might be worth capturing data more frequently and sending # something like average or median values. status['openbts_load'] = {} try: status['openbts_load'] = bts.get_load() except BSSError as e: logger.error("bts get_load error: %s" % e) for key, val in list(self._checkin_load_stats.items()): status['openbts_load']['checkin.' + key] = val self._checkin_load_stats.clear() try: status['openbts_noise'] = bts.get_noise() except BSSError as e: logger.error("bts get_noise error: %s" % e) status['radio'] = {} try: status['radio']['band'] = bts.get_band() # eventually need to also grab all used channels, not just c0 # TODO: (kheimerl) T13270338 Add multiband support status['radio']['c0'] = bts.get_arfcn_c0() #also add power here eventually # TODO: (kheimerl) T13270365 Add power level support except BSSError as e: #delete the key if this failed del status['radio'] logger.error("bts radio error: %s" % e) # Add balance sync data status['subscribers'] = subscriber.get_subscriber_states( imsis=events.EventStore().modified_subs()) # Add delta protocol context (if available) to let server know, # client supports delta optimization & has a prior delta state if delta.DeltaProtocol.CTX_KEY not in status: # just a precaution sections_ctx = {} for section, ctx in list(CheckinHandler.section_ctx.items()): if ctx: sections_ctx[section] = ctx.to_proto_dict() if sections_ctx: status[delta.DeltaProtocol.CTX_KEY] = { delta.DeltaProtocolOptimizer.SECTIONS_CTX_KEY: sections_ctx } # Send checkin request. uuid = snowflake.snowflake() data = { 'status': status, 'bts_uuid': uuid, } headers = dict(self.auth_header) # Set content type to app/json & utf-8, compressed or not - JSON should # be more efficient then URL encoded JSON form payload headers['Content-Type'] = 'application/json; charset=utf-8' data_json = json.dumps(data) decompressed_status_len = len(data_json) status_len = decompressed_status_len if status_len > endaga_ic.MIN_COMPRESSIBLE_REQUEST_SZ: # try to gzip payload, send uncompressed if compression failed try: gzbuf = BytesIO() with GzipFile(mode='wb', fileobj=gzbuf) as gzfile: gzfile.write(bytes(data_json, encoding='UTF-8')) data_json = gzbuf.getvalue() # Using Content-Encoding header since AWS cannot handle # Transfer-Encoding header which would be more appropriate here headers['Content-Encoding'] = 'gzip' status_len = len(data_json) # set len to reflect compression except BaseException as e: logger.error("Checkin request Gzip error: %s" % e) headers['Content-Length'] = str(status_len) post_start = time.time() try: r = self.session.post( self.conf['registry'] + "/checkin?id=" + # add part of uuid to the query, it helps with # debugging & server side logging and can # be used by LBs uuid[:8], headers=headers, data=data_json, timeout=timeout, cookies=self._session_cookies) except BaseException as e: logger.error("Endaga: checkin failed , network error: %s." % e) self._cleanup_session() self._checkin_load_stats['req_sz'] = status_len self._checkin_load_stats['raw_req_sz'] = decompressed_status_len self._checkin_load_stats['post_lat'] = time.time() - post_start raise post_end = time.time() # Make sure either server sent charset or we set it to utf-8 (JSON # default) if not r.encoding: r.encoding = 'utf-8' text = r.text decompressed_response_len = len(text) response_len = decompressed_response_len # Try to get correct content length from HTTP headers, it should # reflect correctly compressed length. if it fails - fall back to # getting length of returned text cont_len = r.headers.get('Content-Length') if cont_len: try: response_len = int(cont_len) except BaseException: pass if r.status_code == 200: try: CheckinHandler(text) logger.info("Endaga: checkin success.") if r.cookies is not None: if self._session_cookies is None: # First time cookies are seen from server # initialize the cookies dict self._session_cookies = dict(r.cookies) else: for key, value in r.cookies.items(): # if server sent new/updated cookies, update them, # but keep previously set cokies as well. ELBs # do not send AWSELB cookies on every request & # expect clients to 'remember' them self._session_cookies[key] = value except BaseException: self._cleanup_session() raise else: logger.error("Endaga: checkin failed (%d), reason: %s, body: %s" % (r.status_code, r.reason, r.text)) # cleanup session on any error if r.status_code >= 300: self._cleanup_session() checkin_end = time.time() self._checkin_load_stats['req_sz'] = status_len # request payload SZ self._checkin_load_stats['raw_req_sz'] = decompressed_status_len self._checkin_load_stats[ 'rsp_sz'] = response_len # response payload SZ self._checkin_load_stats['raw_rsp_sz'] = decompressed_response_len # Checkin Latencies self._checkin_load_stats['post_lat'] = post_end - post_start self._checkin_load_stats['process_lat'] = checkin_end - post_end self._checkin_load_stats['lat'] = checkin_end - checkin_start data['response'] = {'status': r.status_code, 'text': r.text} return data
def post(self, request, format=None): """Handle POST requests.""" # First make sure the dom parses. try: dom = xml.dom.minidom.parseString(request.POST['cdr']) except xml.parsers.expat.ExpatError: logger.warning("invalid XML CDR: '%s'" % (request.POST['cdr'], )) return Response("Bad XML", status=status.HTTP_400_BAD_REQUEST) except KeyError: logger.warning("invalid POST request: '%s'" % (request.POST, )) return Response("Missing CDR", status=status.HTTP_400_BAD_REQUEST) # Then make sure all of the necessary pieces are there. Fail if any # required tags are missing data = {} for tag_name in [ "billsec", "username", "caller_id_name", "network_addr", "destination_number" ]: data[tag_name] = dom.getElementsByTagName(tag_name) for tag_name in data: if not data[tag_name]: return Response("Missing XML", status=status.HTTP_400_BAD_REQUEST) else: data[tag_name] = self.getText(data[tag_name][0].childNodes) # Convert certain tags to ints. for tag_name in ["billsec"]: data[tag_name] = int(data[tag_name]) # Try to get Number instances for both the caller and recipient. # If we find both Numbers in the system then a user on one Endaga # network is calling a subscriber on a different Endaga-managed # network, cool. In our cloud freeswitch instance we actually # "short circuit" this call so it never goes to Nexmo, but to the # operators the call is a regular incoming / outgoing event, so we # will bill it as such. caller_number, dest_number = None, None try: caller_number = Number.objects.get(number=data["caller_id_name"]) try: caller_cost = caller_number.network.calculate_operator_cost( 'off_network_send', 'call', destination_number=data['destination_number']) except ValueError as ex: # this is raised iff the destination has an invalid prefix logger.error("invalid number prefix: %s" % (ex, )) caller_cost = 0 caller_number.network.bill_for_call(caller_cost, data['billsec'], 'outside_call') logger.info("billed network '%s' for outgoing call: %d" % (caller_number.network.name, caller_cost)) except Number.DoesNotExist: pass try: dest_number = Number.objects.get(number=data["destination_number"]) # cost to receive doesn't take source/caller into account dest_cost = dest_number.network.calculate_operator_cost( 'off_network_receive', 'call') dest_number.network.bill_for_call(dest_cost, data['billsec'], 'incoming_call') logger.info("billed network '%s' for incoming call: %d" % (dest_number.network.name, dest_cost)) except Number.DoesNotExist: pass if not (caller_number or dest_number): # We didn't find either Number, that's a failure. return Response("Invalid caller and destination", status=status.HTTP_404_NOT_FOUND) # Local, incoming and outgoing calls respond with 200 OK. return Response("", status=status.HTTP_200_OK)
def GET(self, uuid): logger.info('GET mock number allocation with env: %s' % str(web.ctx.env)) auth_token = web.ctx.env['HTTP_AUTHORIZATION'] assert uuid == snowflake.snowflake() assert auth_token == "Token %s" % CONF['endaga_token'] return '1%010d' % random.randint(1000000000, 9999999999)
def GET(self): logger.info('GET mock bts register') return web.forbidden()
def _create_event(imsi, old_credit, new_credit, reason, kind=None, call_duration=None, billsec=None, from_imsi=None, from_number=None, to_imsi=None, to_number=None, tariff=None, up_bytes=None, down_bytes=None, timespan=None, write=True): """Logs a generic UsageEvent in the EventStore. Also writes this action to logger. Args: imsi: the IMSI connected to this event old_credit: the account's balance before this action new_credit: the account's balance after this action reason: a string describing this event kind: the type of event. If None, we will attempt to lookup the type based on the reason. call_duration: duration, including connect, if it was a call (seconds) billsec: billable duration of the event if it was a call (seconds) from_imsi: sender IMSI from_number: sender number to_imsi: destination IMSI to_number: destination number tariff: the cost per unit applied during this transaction up_bytes: integer amount of data uploaded during the timespan down_bytes: integer amount of data downloaded during the timespan timsespan: number of seconds over which this measurement was taken write: write event to the eventstore (default: True; only for tests) Returns: A dictionary representing the event """ template = ('new event: user: %s, old_credit: %d, new_credit: %d,' ' change: %d, reason: %s\n') message = template % (imsi, old_credit, new_credit, new_credit - old_credit, reason) logger.info(message) # Add this event to the DB. This is the canonical definition of a # UsageEvent. # Version 5: added up_bytes, down_bytes and timespan # Version 4: added billsec, call_duration (removed underscore) # Version 3: ~~a mystery~~ # Version 2: added 'call duration', from_imsi, to_imsi, from_number, # to_number # Version 1: date, imsi, oldamt, newamt, change, reason, kind if not kind: kind = kind_from_reason(reason) data = { 'date': time.strftime('%Y-%m-%d %H:%M:%S'), 'imsi': imsi, 'oldamt': old_credit, 'newamt': new_credit, 'change': new_credit - old_credit, 'reason': reason, 'kind': kind, 'call_duration': call_duration, 'billsec': billsec, 'from_imsi': from_imsi, 'to_imsi': to_imsi, 'from_number': from_number, 'to_number': to_number, 'tariff': tariff, 'up_bytes': up_bytes, 'down_bytes': down_bytes, 'timespan': timespan, 'version': 5, } # TODO(shasan): find a way to remove this and mock out in testing instead if write: event_store = EventStore() event_store.add(data) return data