def __init__(self):
self._conf = ConfigDB()
# initialise logging level from DB (if set - otherwise 'warning')
# NB - this is the ONLY time changes to the log level are actually
# passed to the logging framework
log_level = self._conf.get("logger.global.log_level", "warning")
logger.DefaultLogger.update_handler(level=log_level)
logger.notice("EndagaD started")
def try_to_autoupgrade():
"""The gatekeeper of the upgrade_endaga method.
Autoupgrades can be configured to run as soon as new software is available
via the autoupgrade.in_window configdb key. This method will invoke
upgrade_endaga if autoupgrades are enabled. If windowed upgrades are
enabled, this will check if it's the right time and if an upgrade hasn't
been run recently (default for "recently" is the last ten minutes).
"""
conf = ConfigDB()
window_duration = 10 * conf.get('registration_interval', 60)
last_upgrade_format = '%Y-%m-%d %H:%M:%S'
# Do nothing if autoupgrades are disabled.
if not conf.get('autoupgrade.enabled', False):
return
# Also do nothing if there is no new metapackage available. This info is
# propagated for the beta and stable channels via the checkin response.
channel = conf.get('autoupgrade.channel', 'stable')
key = 'autoupgrade.latest_%s_version' % channel
available_version = sortable_version(conf.get(key, '0.3.29'))
installed_version = sortable_version(conf['endaga_version'])
if available_version <= installed_version:
return
# If we're configured to only upgrade in a window (as opposed to as soon as
# a new package is available), we need some additional checks.
if conf.get('autoupgrade.in_window', False):
# Do nothing if we've already performed an upgrade recently.
last_upgrade = conf.get('autoupgrade.last_upgrade',
'2015-07-14 02:30:15')
last_upgrade = datetime.strptime(last_upgrade, last_upgrade_format)
now = datetime.utcnow()
delta = (now - last_upgrade).total_seconds()
if delta < window_duration:
return
# See if we're in the upgrade window. Get the current time and the
# window_start time as datetimes. These are weird "date-less"
# datetimes -- both dates will be 1-1-1900 but the part that we care
# about, the times, will be comparable.
window_format = '%H:%M:%S'
now = datetime.strptime(now.strftime(window_format), window_format)
window_start = conf.get('autoupgrade.window_start', '02:30:00')
window_start = datetime.strptime(window_start, window_format)
# Fail if we're currently before or after the window.
if now < window_start:
return
if now > window_start + timedelta(seconds=window_duration):
return
# All checks pass, perform the upgrade and save the last upgraded time.
upgrade_endaga(conf.get('autoupgrade.channel', 'stable'))
conf['autoupgrade.last_upgrade'] = datetime.strftime(
datetime.utcnow(), last_upgrade_format)
def get_vpn_ip():
"""
Gets the system VPN IP, or returns None if no VPN is up.
"""
try:
# Pylint gets confused about the contents of this package for some
# reason.
# pylint: disable=no-member
conf = ConfigDB()
ifname = conf.get('external_interface') # vpn_interface?
assert ifname is not None, \
"VPN interface ('external_interface') not configured"
return netifaces.ifaddresses(ifname)[netifaces.AF_INET][0]['addr']
except Exception:
return None
def __init__(self, user, port, call_handler, sms_handler,
self_ip="127.0.0.1", other_ip="127.0.0.1"):
#add user to the fakebts list of camped subscribers
self.conf = ConfigDB()
if 'fakebts.camped' not in self.conf:
self.conf['fakebts.camped'] = json.dumps([])
self.conf['fakebts.camped'] = json.dumps(json.loads(self.conf['fakebts.camped']) + [user])
def main():
"""Main routine run by gprsd."""
# Initialize some timestamps and connect to the ConfigDB.
now = time.time()
last_scrape_time = now
last_event_generation_time = now
last_removal_of_old_data = now
config_db = ConfigDB()
while True:
now = time.time()
# Get GPRS usage data and store it in the DB.
if (now - last_scrape_time >
config_db['gprsd_cli_scrape_period']):
last_scrape_time = now
utilities.gather_gprs_data()
# Generate events for the EventStore with data from the GPRS table.
if (now - last_event_generation_time >
config_db['gprsd_event_generation_period']):
last_event_generation_time = now
start_time = now - config_db['gprsd_event_generation_period']
utilities.generate_gprs_events(start_time, now)
# Clean old records out of the GPRS table.
if (now - last_removal_of_old_data >
config_db['gprsd_cleanup_period']):
last_removal_of_old_data = now
utilities.clean_old_gprs_records(
now - config_db['gprsd_max_data_age'])
# Wait for a bit, then do it again.
time.sleep(SLEEP_TIME)
def __init__(self):
self.conf = ConfigDB()
self.subscribers = Subscribers(
host=self.conf['bts.osmocom.ip'],
port=self.conf['bts.osmocom.bsc_vty_port'],
hlr_loc=self.conf['bts.osmocom.hlr_loc'],
timeout=self.conf['bss_timeout'])
def __init__(self):
self.conf = ConfigDB()
self.defaults = {
'sddch': 8,
'tchf': 4,
'pch': 2,
'agch': 2,
'pdch': 2,
'mnc': "001",
'mnc': "01",
'c0': 51,
'band': "GSM900",
'shortName': "fakeBTS",
'openRegistration': ".*",
'timer.3212': 6,
'camped': json.dumps([]),
}
def setUpClass(cls):
"""Setup the test app."""
cls.test_app = TestApp(core.federer.app.wsgifunc())
cls.endpoint = '/config/deactivate_subscriber'
# Setup a serializer so we can send signed data. Bootstrap the secret.
config_db = ConfigDB()
config_db['bts_secret'] = 'yup'
cls.serializer = itsdangerous.JSONWebSignatureSerializer(
config_db['bts_secret'])
def get_vpn_ip():
"""
Gets the system VPN IP, or returns None if no VPN is up.
"""
try:
# Pylint gets confused about the contents of this package for some
# reason.
# pylint: disable=no-member
conf = ConfigDB()
ifname = conf['external_interface']
return netifaces.ifaddresses(ifname)[netifaces.AF_INET][0]['addr']
except:
return None
def setUpClass(cls):
"""Setup the ConfigDB with a fake secret, and mock other items."""
cls.config_db = ConfigDB()
cls.config_db['bts_secret'] = 'test-secret-123'
cls.original_logger = interconnect.logger
cls.mock_logger = mock.Mock()
interconnect.logger = cls.mock_logger
# Mock bts for TMSIs
cls.original_bts = interconnect.bts
interconnect.bts = mocks.MockBTS()
# Mock subscriber
cls.original_subscriber = interconnect.subscriber
interconnect.subscriber = mocks.MockSubscriber()
def __init__(self):
self.conf = ConfigDB()
self.subscribers = Subscribers(
host=self.conf['bts.osmocom.ip'],
port=self.conf['bts.osmocom.bsc_vty_port'],
hlr_loc=self.conf['bts.osmocom.hlr_loc'],
timeout=self.conf['bss_timeout'])
self.network = Network(host=self.conf['bts.osmocom.ip'],
port=self.conf['bts.osmocom.bsc_vty_port'],
timeout=self.conf['bss_timeout'])
self.bts = BTS(host=self.conf['bts.osmocom.ip'],
port=self.conf['bts.osmocom.bsc_vty_port'],
timeout=self.conf['bss_timeout'])
self.trx = TRX(host=self.conf['bts.osmocom.ip'],
port=self.conf['bts.osmocom.bsc_vty_port'],
timeout=self.conf['bss_timeout'])
def __init__(self,
user,
port,
call_handler,
sms_handler,
self_ip="127.0.0.1",
other_ip="127.0.0.1"):
BaseFakePhone.__init__(self,
user,
port,
call_handler,
sms_handler,
self_ip=self_ip,
other_ip=other_ip)
self.user = user
self.conf = ConfigDB()
self.port = port
self.sms_h = sms_handler
self.call_h = call_handler
self.self_ip = self_ip
self.other_ip = other_ip
class EndagaD(object):
"""
Thin wrapper around the main loop that communicates with the cloud
service. Manages registration, checkin and associated responses.
"""
def __init__(self):
self._conf = ConfigDB()
# initialise logging level from DB (if set - otherwise 'warning')
# NB - this is the ONLY time changes to the log level are actually
# passed to the logging framework
log_level = self._conf.get("logger.global.log_level", "warning")
logger.DefaultLogger.update_handler(level=log_level)
logger.notice("EndagaD started")
def _reset_bts_config(self):
logger.notice("Performing set_factory")
try:
if bts.set_factory_config():
logger.notice("Restarting BTS")
bts.restart()
Service.SystemService("freeswitch").restart()
except BSSError as e:
logger.error("bts is probably down: %s" % e)
except Exception as e:
# OSError, IOError or whatever envoy will raise
logger.critical("something unexpected happened: %s" % e)
def run(self):
"""
Main loop for endagad. This moves the system through the various
states of operation -- it should be a state machine really!
General flow is:
1) Tries to get configuration from server to produce VPN keys
2) Generates keys locally.
3) Sends CSR for signing, returns that.
4) Starts system services (FS, BTS, etc) and configures them
appropriately. Note configuration can change depending on registration
and VPN state of the system.
5) Runs checkin periodically.
"""
eapi = interconnect.endaga_ic(self._conf)
if 'registration_interval' not in self._conf:
self._conf['registration_interval'] = 60
UNHEALTHY_THRESH = self._conf.get('bts.unhealthy_threshold', 3)
unhealthy_count = UNHEALTHY_THRESH # fail quickly on first pass
while True:
# Retrieve keys/tokens, or do nothing if we have them.
logger.notice("Performing gen_keys")
registration.generate_keys()
# generate_keys() loads auth token on success. Need to update the
# interconnect client's token if so.
if eapi.token is None:
eapi.token = self._conf['endaga_token']
# Try to register/get VPN credentials. Tries forever if fails.
logger.notice("Performing register")
registration.register(eapi)
# Registered, start services and tries to start VPN. Stop
# everything otherwise.
logger.notice("Performing clear_pid")
registration.clear_old_pid()
logger.notice("Performing update_vpn")
registration.update_vpn()
# At this point, all services should be up, so we can perform
# additional configuration.
self._reset_bts_config()
# Update the inbound_url if the VPN is up.
if system_utilities.get_vpn_ip() is not None:
logger.notice("Performing register_update")
registration.register_update(eapi)
logger.notice("Performing ensure_fs_external_bound")
registration.ensure_fs_external_bound_to_vpn()
# Send checkin to cloud
try:
# Sends events, tries to get config info. Can proceed w/o VPN.
logger.notice("Performing checkin.")
checkin_data = eapi.checkin(timeout=30)
logger.notice("Performing system health check.")
if not registration.system_healthcheck(checkin_data):
unhealthy_count += 1
logger.notice("System unhealthy: %d" % unhealthy_count)
else:
unhealthy_count = 0
except (ConnectionError, Timeout):
logger.error(
"checkin failed due to connection error or timeout.")
except BSSError as e:
logger.error("bts exception: %s" % e)
if unhealthy_count > UNHEALTHY_THRESH:
logger.notice("BTS seems unhealthy, restarting BTS services.")
bts.restart()
Service.SystemService("freeswitch").restart()
# Upgrade the endaga metapackage, when appropriate and only if that
# feature is enabled.
logger.notice("Performing autoupgrade")
system_utilities.try_to_autoupgrade()
# Sleep for some amount of time before retrying
logger.notice("Performing sleep")
time.sleep(self._conf['registration_interval'])
def __init__(self):
self.conf = ConfigDB()
self.smqueue = openbts.components.SMQueue(
socket_timeout=self.conf['bss_timeout'],
cli_timeout=self.conf['bss_timeout'])
import gettext
import traceback
import itsdangerous
from ccm.common import logger
from core import billing
from core import events
from core import freeswitch_interconnect
from core.subscriber import subscriber
from core.config_database import ConfigDB
from core.message_database import MessageDB
cdb = ConfigDB()
gt = gettext.translation(
"endaga", cdb['localedir'], [cdb['locale'], "en"]).gettext
# Hardcode the dashboard's from_number.
DASHBOARD_FROM_NUMBER = '0000'
class incoming(object):
def __init__(self):
self.conf = cdb
self.fs_ic = freeswitch_interconnect.freeswitch_ic(self.conf)
self.tariff_type = "off_network_receive"
self.msgid_db = MessageDB()
def bill(self, to_number, from_number):
try:
# Copyright (c) 2016-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from core.config_database import ConfigDB
#
# Override stubs with system specific implementation
#
conf = ConfigDB()
if conf['bts.type'] == 'fake':
from . import _fakehlr
subscriber = _fakehlr.FakeSubscriberDB()
elif conf['bts.type'] == 'osmocom':
from . import _osmocom
subscriber = _osmocom.OsmocomSubscriber()
else:
from . import _openbts
subscriber = _openbts.OpenBTSSubscriber()
class CheckinHandler(object):
CONFIG_SECTION = "config"
EVENTS_SECTION = "events"
SUBSCRIBERS_SECTION = "subscribers"
# NOTE: Keys in section_ctx dictionary below must match the keys of
# optimized checkin sections: "config", "events", "subscribers", etc.
section_ctx = {
CONFIG_SECTION: delta.DeltaProtocolCtx(),
# Note: EVENTS_SECTION is not optimized
SUBSCRIBERS_SECTION: delta.DeltaProtocolCtx(),
}
def __init__(self, response):
self.conf = ConfigDB()
self.eventstore = EventStore()
r = self.validate(response)
self.process(r)
def process(self, resp_dict):
"""Process sections of a checkin response.
Right now we have three sections: config, events, and subscribers.
"""
if 'status' in resp_dict and resp_dict['status'] == 'deregistered':
reset_registration()
for section in resp_dict:
if section == CheckinHandler.CONFIG_SECTION:
self.process_config(resp_dict[section])
elif section == CheckinHandler.EVENTS_SECTION:
self.process_events(resp_dict[section])
elif section == CheckinHandler.SUBSCRIBERS_SECTION:
self.process_subscribers(resp_dict[section])
elif section != 'status':
logger.error("Unexpected checkin section: %s" % section)
def validate(self, response):
"""Validates a response.
Args:
response: decoded json response from the server as a python
dictionary.
Returns: a python dictionary containing the checkin response, otherwise
throws errors.
"""
r = json.loads(response)
return r['response']
@delta.DeltaCapable(section_ctx['config'], True)
def process_config(self, config_dict):
for section in config_dict:
if section == "endaga":
self.conf.process_config_update(config_dict[section])
# TODO cloud should use generic key names not openbts specific
elif section == "openbts":
bts.process_bts_settings(config_dict[section])
elif section == "prices":
process_prices(config_dict['prices'], self.conf)
elif section == "autoupgrade":
self.process_autoupgrade(config_dict['autoupgrade'])
# wrap the subscriber method in order to keep delta context encapsulated
@delta.DeltaCapable(section_ctx['subscribers'], True)
def process_subscribers(self, data_dict):
subscriber.process_update(data_dict)
def process_events(self, data_dict):
"""Process information about events.
Right now, there should only be one value here: seqno, which denotes
the highest seqno for this BTS for which the server has ack'd.
"""
if "seqno" in data_dict:
seqno = int(data_dict['seqno'])
self.eventstore.ack(seqno)
def process_autoupgrade(self, data):
"""Process information about autoupgrade preferences.
Args:
data: a dict of the form {
'enabled': True,
'channel': 'dev',
'in_window': True, # whether to upgrade in a window or not. If
# not, this means we should upgrade as soon as
# new packages are available.
'window_start': '02:45:00'
'latest_stable_version': '1.2.3',
'latest_beta_version': '5.6.7',
}
The configdb keys are prefixed with "autoupgrade." (e.g.
autoupgrade.enabled).
"""
for key in ('enabled', 'channel', 'in_window', 'window_start',
'latest_stable_version', 'latest_beta_version'):
configdb_key = 'autoupgrade.%s' % key
# Set the value if it's not already in the config db or if it's
# changed.
existing_value = self.conf.get(configdb_key, None)
if existing_value != data[key]:
self.conf[configdb_key] = data[key]
def setUpClass(cls):
cls.config_db = ConfigDB()
cls.key = 'test-key'
break
time.sleep(1)
def call_handler(fp):
print ("Call Received")
fp.call_received = True
def sms_handler(fp, content):
print ("SMS Received: " + content)
fp.sms_received = True
fp.content = content
CONF = ConfigDB()
FEDERER_TIMEOUT = 15 # seconds
SMQ_TIMEOUT = 15 # seconds
FS_TIMEOUT = 15 # seconds
class FakePhoneTest_01_Nominal(unittest.TestCase):
"""Testing FS functionality"""
@classmethod
def setUpClass(cls):
# Mock the Endaga API endpoint in a new process.
cls.webserver = Thread(
target=core.tests.mock_api.APP_NOMINAL.run)
# This is a kludge -- web.py is strict on taking the port from argv.
sys.argv = ['8080']
def __init__(self):
super(OpenBTSSubscriber, self).__init__()
self.conf = ConfigDB()
self.sip_auth_serve = openbts.components.SIPAuthServe(
socket_timeout=self.conf['bss_timeout'],
cli_timeout=self.conf['bss_timeout'])
def __init__(self):
self.ic = None
self.conf = ConfigDB()
self.worker = self.registration_worker
self.fs_ic = freeswitch_interconnect.freeswitch_ic(self.conf)
def __init__(self):
self.conf = ConfigDB()
self.msgid_db = MessageDB()
self.ic = interconnect.endaga_ic(self.conf)
class FakeBTS(BaseBTS):
def __init__(self):
self.conf = ConfigDB()
self.defaults = {
'sddch': 8,
'tchf': 4,
'pch': 2,
'agch': 2,
'pdch': 2,
'mnc': "001",
'mnc': "01",
'c0': 51,
'band': "GSM900",
'shortName': "fakeBTS",
'openRegistration': ".*",
'timer.3212': 6,
'camped': json.dumps([]),
}
def __get(self, name):
db_name = "fakebts." + name
if name in self.defaults:
return self.conf.get(db_name, default=self.defaults[name])
else:
return self.conf.get(db_name)
def __set(self, name, value):
self.conf['fakebts.' + name] = value
def set_factory_config(self):
""" Done. """
pass
def get_camped_subscribers(self, access_period=0, auth=1):
#camped is serialized
camped = json.loads(self.__get('camped'))
#not a real user, but we always need one
camped += ['IMSI001010000000000']
res = []
for camp in camped:
res.append({
'IMSI': camp,
'ACCESSED': time.time(),
})
return res
def get_load(self):
return {
'sdcch_load': random.choice(range(0, self.__get('sddch'))),
'sdcch_available': self.__get('sddch'),
'tchf_load': random.choice(range(0, self.__get('tchf'))),
'tchf_available': self.__get('tchf'),
'pch_active': random.choice(range(0, self.__get('pch'))),
'pch_total': self.__get('pch'),
'agch_active': random.choice(range(0, self.__get('agch'))),
'agch_pending': 0,
'gprs_current_pdchs': random.choice(range(0, self.__get('pdch'))),
#probably should math this
'gprs_utilization_percentage': .4,
}
def get_noise(self):
return {
'noise_rssi_db': 60,
'noise_ms_rssi_target_db': 80,
}
def set_mcc(self, mcc):
self.__set('mcc', mcc)
def set_mnc(self, mnc):
self.__set('mnc', mnc)
def set_short_name(self, short_name):
self.__set('shortName', short_name)
def set_open_registration(self, expression):
self.__set('openRegistration', expression)
def set_timer(self, timer, value):
self.__set('timer.' + timer, value)
def set_band(self, band):
self.__set('band', band)
def set_arfcn_c0(self, arfcn):
self.__set('c0', arfcn)
def get_mcc(self):
return self.__get('mcc')
def get_mnc(self):
return self.__get('mnc')
def get_short_name(self):
return self.__get('shortName')
def get_open_registration(self):
return self.__get('openRegistration')
def get_timer(self, timer):
try:
return self.__get('timer.' + timer)
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_available_bands(self):
return [self.get_band()]
def get_available_arfcns(self):
return [self.get_arfcn_c0()]
def get_band(self):
return self.__get('band')
def get_arfcn_c0(self):
return self.__get('c0')
def get_versions(self):
#custom keys for this BTS type
versions = BaseBTS.get_versions(self)
versions['fakebts'] = self.conf['gsm_version']
return versions
def __init__(self):
self.conf = ConfigDB()
self.openbts = openbts.components.OpenBTS(
socket_timeout=self.conf['bss_timeout'],
cli_timeout=self.conf['bss_timeout'])
def __init__(self):
self.conf = ConfigDB()
self.interconnect_client = interconnect.endaga_ic(self.conf)
self.worker = self.sms_worker
def __init__(self, response):
self.conf = ConfigDB()
self.eventstore = EventStore()
r = self.validate(response)
self.process(r)
def __init__(self):
self._conf = ConfigDB()
def setUpClass(cls):
"""Generate one set of checkin data to be analyzed in the tests."""
# Mock the requests module so we don't actually POST.
cls.original_requests = core.interconnect.requests
cls.mock_requests = mocks.MockRequests(200)
core.interconnect.requests = cls.mock_requests
# Mock subscriber
cls.original_subscriber = core.interconnect.subscriber
core.interconnect.subscriber = mocks.MockSubscriber()
# Mock core.events for generating usage events and handling the checkin
# response.
cls.original_events = core.interconnect.events
core.interconnect.events = mocks.MockEvents()
# Mock snowflake.
cls.original_snowflake = core.interconnect.snowflake
cls.mock_uuid = '09031a16-6361-4a93-a934-24c990ef4b87'
core.interconnect.snowflake = mocks.MockSnowflake(cls.mock_uuid)
# Mock BTS for TMSIs
cls.original_bts = core.interconnect.bts
core.interconnect.bts = mocks.MockBTS()
# Mock a lot the package version numbers that should be sent in the
# checkin.
config_db = ConfigDB()
cls.package_versions = {
'endaga': '1.2.3',
'freeswitch': '2.3.4',
'gsm': '3.4.5',
'python-endaga-core': '4.5.6',
'python-gsm': '5.6.7',
}
for key in cls.package_versions:
config_db['%s_version' % key] = cls.package_versions[key]
# Mock psutil for system utilization stats.
cls.original_psutil = core.system_utilities.psutil
utilization = {
'cpu_percent': 20.1,
'memory_percent': 33.3,
'disk_percent': 52.2,
'bytes_sent': 1234,
'bytes_received': 5678,
}
core.system_utilities.psutil = mocks.MockPSUtil(utilization)
# Create some fake events
es = core.interconnect.events.EventStore()
es.add(
cls.original_events._create_event(imsi="IMSI123",
old_credit=0,
new_credit=100,
reason="Foo",
write=False))
es.add(
cls.original_events._create_event(imsi="IMSI123",
old_credit=0,
new_credit=100,
reason="Foo",
write=False))
es.add(
cls.original_events._create_event(imsi="IMSI321",
old_credit=0,
new_credit=100,
reason="Foo",
write=False))
# Setup a secret key.
config_db['bts_secret'] = cls.mock_uuid
# Attempt a checkin.
config_db = core.config_database.ConfigDB()
cls.endaga_ic = core.interconnect.endaga_ic(config_db)
cls.endaga_ic.checkin()
# Get the POSTed data and a deserialized form for convenience.
cls.data = cls.mock_requests.post_data
cls.deserialized_status = json.loads(cls.data['status'])