def init_agent():
services, workers, session, _ = parse_old_config()
endpoint = Service('cmd_responder', EventSetOutputConnector('cmd_responder').send,
StateManager.save_dialog_dict, 1, ['responder'])
input_srv = Service('input', None, StateManager.add_human_utterance_simple_dict, 1, ['input'])
pipeline = Pipeline(services)
pipeline.add_responder_service(endpoint)
pipeline.add_input_service(input_srv)
agent = Agent(pipeline, StateManager())
return agent, session
def update_vpn():
"""
If the BTS is registered, try to start the VPN. If the BTS is not
registered, skip.
If the BTS is unregistered (on the dashboard), no services are available.
Regardless of whether the VPN is up or down, all services should be started
(this will enable disconnected operation). However, when the VPN comes
online, we need to restart FS to make sure that we're bound to the VPN IP
so outgoing calls can work.
"""
if not ('bts_registered' in conf and conf['bts_registered']):
logger.error('BTS is not yet registered, skipping VPN setup, killing'
' all services.')
for s in SERVICES:
if s.name == 'endagad':
continue
s.stop()
return
# If the VPN is down, try to start it, then restart FS if we succeed.
if not system_utilities.get_vpn_ip():
max_attempts = 10
for _ in range(0, max_attempts):
# Sometimes the vpn service is started, but the VPN is still down.
# If this is the case, stop the vpn service first.
openvpn_service = Service.SystemService('ccm-openvpn')
if openvpn_service.status() == ServiceState.Running:
openvpn_service.stop()
if openvpn_service.start():
logger.notice('VPN service started')
if system_utilities.get_vpn_ip():
logger.notice('VPN up - restarting freeswitch')
Service.SystemService('freeswitch').restart()
else:
logger.error('VPN interface (%s) is down' %
conf.get('external_interface'))
else:
logger.error(
'VPN failed to start after registration, retrying.')
time.sleep(3)
if not system_utilities.get_vpn_ip():
logger.error('Failed to set up VPN after %d attempts!' %
max_attempts)
# Start all the other services. This is safe to run if services are
# already started.
for s in SERVICES:
try:
s.start()
except Exception as e:
logger.critical("Exception %s while starting %s" % (e, s.name))
def restart(self):
"""An openbts specific command to restart the bts."""
delegator.run("sudo killall transceiver")
# If the pid file specified doesn't match a running instance of the
# process, remove the PID file. This is a workaround for a recurring
# OpenBTS issue we see. Note, caller must have permissions to remove file.
# Determine PIDs associated with pname
path = "/var/run/OpenBTS.pid"
cmd = delegator.run("ps -A | grep OpenBTS")
if cmd.return_code != 0:
output = ''
else:
output = cmd.out
pids = []
for line in output.split('\n'):
try:
pids.append(int(line.strip().split()[0]))
except ValueError:
continue # not a pid, so ignore it
except IndexError:
continue # malformed, ignore
try:
with open(path, "r") as f:
pid = int(f.read().strip())
if pid not in pids:
os.remove(path)
except IOError as e:
# ignore ENOENT (pid file doesn't exist)
if e.errno != errno.ENOENT:
raise
# now restart openbts
Service.SupervisorService("openbts").restart()
def reset_registration(registry=None):
"""Removes existing registration but does not erase the snowflake UUID."""
if registry:
if not (
(registry.startswith('http://') or registry.startswith('https://'))
and registry.endswith('/api/v1')):
raise ValueError('invalid registry URL: %s' % (registry, ))
conf['registry'] = registry
# Remove the relevant endaga config keys.
del conf['bts_registered'] # registration status
del conf['bts_secret'] # the temporary key for authing requests
_reset_endaga_token() # the account owner's API token
# Remove the vpn configuration and keys.
for f in [
'endaga-client.key', 'endaga-client.crt', 'endaga-client.req',
'endaga-sslconf.conf.noauto', 'endaga-vpn-client.conf.noauto'
]:
fpath = '/etc/openvpn/%s' % f
if os.path.exists(fpath):
os.remove(fpath)
# Stop all other services and restart endagad.
for s in SERVICES:
if s.name == 'endagad':
continue
s.stop()
Service.SystemService('endagad').restart()
def test_service(self):
s = Service()
s.set("wenbinhuang", "liu")
res = s.get("wenbinhuang")
print(res)
self.assertEqual(res == "liu", True)
s.delete("wenbinhuang")
self.assertEqual(s.get("wenbinhuang") == None, True)
def update_vpn():
"""
If the BTS is registered, try to start the VPN. If the BTS is not
registered, skip.
If the BTS is unregistered (on the dashboard), no services are available.
Regardless of whether the VPN is up or down, all services should be started
(this will enable disconnected operation). However, when the VPN comes
online, we need to restart FS to make sure that we're bound to the VPN IP
so outgoing calls can work.
"""
if not ('bts_registered' in conf and conf['bts_registered']):
logger.error("BTS is not yet registered, skipping VPN setup, killing"
" all services.")
for s in SERVICES:
if s.name == "endagad":
continue
s.stop()
return
# If the VPN is down, try to start it, then restart FS if we succeed.
if not system_utilities.get_vpn_ip():
max_attempts = 10
for _ in range(0, max_attempts):
# Sometimes the vpn service is started, but the VPN is still down.
# If this is the case, stop the vpn service first.
openvpn_service = Service.SupervisorService("openvpn")
if openvpn_service.status() == ServiceState.Running:
openvpn_service.stop()
if (openvpn_service.start() and system_utilities.get_vpn_ip()):
logger.notice("VPN up restarting services")
Service.SystemService("freeswitch").restart()
else:
logger.error("VPN didn't come up after registration,"
" retrying.")
time.sleep(3)
if not system_utilities.get_vpn_ip():
logger.error("Failed to set up VPN after %d attempts!" %
max_attempts)
# Start all the other services. This is safe to run if services are
# already started.
for s in SERVICES:
s.start()
def ensure_fs_external_bound_to_vpn():
# Make sure that we're bound to the VPN IP on the external sofia profile,
# assuming the VPN is up.
vpn_ip = system_utilities.get_vpn_ip()
if not vpn_ip:
return
external_profile_ip = system_utilities.get_fs_profile_ip('external')
if external_profile_ip != vpn_ip: # TODO: treat these as netaddr, not string
logger.warning('external profile should be bound to VPN IP and isn\'t,'
' restarting FS.')
Service.SystemService('freeswitch').restart()
def main():
services, workers, session = parse_old_config()
for s in services:
if 'RESPONSE_SELECTORS' in s.tags:
continue
if s.is_sselector():
s.connector_func = DummySelectorConnector(['chitchat', 'odqa'], 0.01, s.name).send
else:
s.connector_func = DummyConnector(['we have a phrase', 'and another one', 'not so short one'], 0.01,
s.name).send
intermediate_storage = {}
endpoint = Service('http_responder', HttpOutputConnector(intermediate_storage, 'http_responder').send,
StateManager.save_dialog_dict, 1, ['responder'])
input_srv = Service('input', None, StateManager.add_human_utterance_simple_dict, 1, ['input'])
register_msg, process_callable = prepare_agent(services, endpoint, input_srv, args.response_logger)
app = init_app(register_msg, intermediate_storage, prepare_startup(workers, process_callable, session),
on_shutdown)
web.run_app(app, port=args.port)
def run():
from core.agent import Agent
from core.state_manager import StateManager
from core.skill_manager import SkillManager
from core.rest_caller import RestCaller
from core.service import Service
from core.postprocessor import DefaultPostprocessor
from core.response_selector import ConfidenceResponseSelector
from core.skill_selector import ChitchatQASelector
from core.config import MAX_WORKERS, ANNOTATORS, SKILL_SELECTORS, SKILLS
import logging
logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING)
state_manager = StateManager()
anno_names, anno_urls = zip(*[(annotator['name'], annotator['url']) for annotator in ANNOTATORS])
preprocessor = Service(
rest_caller=RestCaller(max_workers=MAX_WORKERS, names=anno_names, urls=anno_urls))
postprocessor = DefaultPostprocessor()
skill_caller = RestCaller(max_workers=MAX_WORKERS)
response_selector = ConfidenceResponseSelector()
ss_names, ss_urls = zip(*[(annotator['name'], annotator['url']) for annotator in SKILL_SELECTORS])
skill_selector = ChitchatQASelector(rest_caller=RestCaller(max_workers=MAX_WORKERS, names=ss_names, urls=ss_urls))
skill_manager = SkillManager(skill_selector=skill_selector, response_selector=response_selector,
skill_caller=skill_caller, profile_handlers=[skill['name'] for skill in SKILLS
if skill.get('profile_handler')])
agent = Agent(state_manager, preprocessor, postprocessor, skill_manager)
def infer(messages: Collection[Message], dialog_ids):
utterances: List[Optional[str]] = [message.text for message in messages]
tg_users: List[User] = [message.from_user for message in messages]
u_tg_ids = [str(user.id) for user in tg_users]
u_tg_data = [{
'id': user.id,
'username': user.username,
'first_name': user.first_name,
'last_name': user.last_name
}
for user in tg_users]
u_d_types = [None] * len(messages)
date_times = [datetime.utcnow()] * len(messages)
locations: List[Optional[Location]] = [message.location for message in messages]
ch_types = ['telegram'] * len(messages)
answers = agent(utterances=utterances, user_telegram_ids=u_tg_ids, user_device_types=u_d_types,
date_times=date_times, locations=locations, channel_types=ch_types)
return answers
return infer
def _reset_bts_config(self):
logger.notice("Performing set_factory")
try:
if bts.set_factory_config():
logger.notice("Restarting BTS")
bts.restart()
Service.SystemService("freeswitch").restart()
except BSSError as e:
logger.error("bts is probably down: %s" % e)
except Exception as e:
# OSError, IOError or whatever envoy will raise
logger.critical("something unexpected happened: %s" % e)
class SimKV():
def __init__(self, directory_name):
if os.path.exists(directory_name) == False:
os.mkdir(directory_name)
self.directory_name = directory_name
self.service = Service(directory_name)
def open(self, directory_name):
handler = SimKV(directory_name)
return handler
def set(self, key, value):
return self.service.set(key, value)
def get(self, key):
return self.service.get(key)
def delete(self, key):
return self.service.delete(key)
def merge(self):
return self.service.merge()
def close(self):
self.service.close()
def parse(self, services_db_data):
"""
<service_name1>
<service_name2>
...
<service_nameN>
"""
self.db = OrderedDict()
tag = 1
for service_name in services_db_data.split("\n"):
service = Service(name=service_name, tag=tag)
self.add_service(service)
tag += 1
def make_service_from_config_rec(conf_record,
sess,
state_processor_method,
tags,
names_previous_services,
gate,
name_modifier=None):
_worker_tasks = []
if name_modifier:
name = name_modifier(conf_record['name'])
else:
name = conf_record['name']
formatter = conf_record['formatter']
batch_size = conf_record.get('batch_size', 1)
url = conf_record['url']
connector_func = None
if conf_record['protocol'] == 'http':
sess = sess or aiohttp.ClientSession()
if batch_size == 1 and isinstance(url, str):
connector_func = HTTPConnector(sess, url, formatter, name).send
else:
queue = asyncio.Queue()
connector_func = AioQueueConnector(
queue).send # worker task and queue connector
if isinstance(url, str):
urls = [url]
else:
urls = url
for u in urls:
_worker_tasks.append(
QueueListenerBatchifyer(sess, u, formatter, name,
queue, batch_size))
elif conf_record['protocol'] == 'AMQP':
gate = gate or prepare_agent_gateway()
connector_func = AgentGatewayToServiceConnector(
to_service_callback=gate.send_to_service,
service_name=name).send
if connector_func is None:
raise ValueError(
f'No connector function is defined while making a service {name}.'
)
_service = Service(name, connector_func, state_processor_method,
batch_size, tags, names_previous_services,
simple_workflow_formatter)
return _service, _worker_tasks, sess, gate
def _parse_service_tag_and_get_service(self):
"""
SERVICE_TAG_START
<service info>
SERVICE_TAG_END
"""
# This could not be present.
line = self._get_clean_line()
if line == SERVICE_TAG_START_PH:
service_tag = self._parse(
start_ph=SERVICE_TAG_START_PH,
end_ph=SERVICE_TAG_END_PH,
object_to_parse_class=ServiceTag,
)
service_name = self.services_database.get_service_name_from_tag(
service_tag.get_tag())
return Service(service_name, service_tag)
else:
return None
class CoreBackgateServiceTestCase(TransactionTestCase):
"""Test the core backgate service."""
databases = {'read', 'write'}
def setUp(self) -> None:
"""Instantiate the service."""
self.service = Service()
@patch.object(grpc, 'insecure_channel')
def test_say_hello(self, _: MagicMock) -> None:
"""Test saying hello."""
# Prepare data.
name = 'Khaleesi, Mother of Dragons, Breaker of Chains'
request = SayHelloRequest(name=name)
# Execute test.
response = self.service.SayHello(request, MagicMock())
# Assert results.
self.assertIn('The guard says:', response.message)
def reset_registration():
"""Removes existing registration but does not erase the snowflake UUID."""
# Remove the relevant endaga config keys.
del conf['bts_registered'] # registration status
del conf['bts_secret'] # the temporary key for authing requests
del conf['sslconf'] # ssl configuration
conf['endaga_token'] = None # the account owner's API token
# Remove the vpn configuration and keys.
for f in [
"endaga-client.key", "endaga-client.crt", "endaga-client.req",
"endaga-sslconf.conf.noauto", "endaga-vpn-client.conf.noauto"
]:
fpath = "/etc/openvpn/%s" % f
if os.path.exists(fpath):
os.remove(fpath)
# Stop all other services and restart endagad.
for s in SERVICES:
if s.name == "endagad":
continue
s.stop()
Service.SupervisorService("endagad").restart()
def parse_old_config():
services = []
worker_tasks = []
session = None
gateway = None
def make_service_from_config_rec(conf_record,
sess,
state_processor_method,
tags,
names_previous_services,
gate,
name_modifier=None):
_worker_tasks = []
if name_modifier:
name = name_modifier(conf_record['name'])
else:
name = conf_record['name']
formatter = conf_record['formatter']
batch_size = conf_record.get('batch_size', 1)
url = conf_record['url']
connector_func = None
if conf_record['protocol'] == 'http':
sess = sess or aiohttp.ClientSession()
if batch_size == 1 and isinstance(url, str):
connector_func = HTTPConnector(sess, url, formatter, name).send
else:
queue = asyncio.Queue()
connector_func = AioQueueConnector(
queue).send # worker task and queue connector
if isinstance(url, str):
urls = [url]
else:
urls = url
for u in urls:
_worker_tasks.append(
QueueListenerBatchifyer(sess, u, formatter, name,
queue, batch_size))
elif conf_record['protocol'] == 'AMQP':
gate = gate or prepare_agent_gateway()
connector_func = AgentGatewayToServiceConnector(
to_service_callback=gate.send_to_service,
service_name=name).send
if connector_func is None:
raise ValueError(
f'No connector function is defined while making a service {name}.'
)
_service = Service(name, connector_func, state_processor_method,
batch_size, tags, names_previous_services,
simple_workflow_formatter)
return _service, _worker_tasks, sess, gate
for anno in ANNOTATORS_1:
service, workers, session, gateway = make_service_from_config_rec(
anno, session, StateManager.add_annotation_dict, ['ANNOTATORS_1'],
set(), gateway)
services.append(service)
worker_tasks.extend(workers)
previous_services = {i.name for i in services if 'ANNOTATORS_1' in i.tags}
if ANNOTATORS_2:
for anno in ANNOTATORS_2:
service, workers, session, gateway = make_service_from_config_rec(
anno, session, StateManager.add_annotation_dict,
['ANNOTATORS_2'], previous_services, gateway)
services.append(service)
worker_tasks.extend(workers)
previous_services = {
i.name
for i in services if 'ANNOTATORS_2' in i.tags
}
if ANNOTATORS_3:
for anno in ANNOTATORS_3:
service, workers, session, gateway = make_service_from_config_rec(
anno, session, StateManager.add_annotation_dict,
['ANNOTATORS_3'], previous_services, gateway)
services.append(service)
worker_tasks.extend(workers)
previous_services = {
i.name
for i in services if 'ANNOTATORS_3' in i.tags
}
if SKILL_SELECTORS:
for ss in SKILL_SELECTORS:
service, workers, session, gateway = make_service_from_config_rec(
ss, session, StateManager.do_nothing,
['SKILL_SELECTORS', 'selector'], previous_services, gateway)
services.append(service)
worker_tasks.extend(workers)
previous_services = {
i.name
for i in services if 'SKILL_SELECTORS' in i.tags
}
if SKILLS:
for s in SKILLS:
service, workers, session, gateway = make_service_from_config_rec(
s, session, StateManager.add_hypothesis_dict, ['SKILLS'],
previous_services, gateway)
services.append(service)
worker_tasks.extend(workers)
previous_services = {i.name for i in services if 'SKILLS' in i.tags}
if not RESPONSE_SELECTORS:
services.append(
Service(
'confidence_response_selector',
ConfidenceResponseSelectorConnector(
'confidence_response_selector').send,
StateManager.add_bot_utterance_simple_dict, 1,
['RESPONSE_SELECTORS'], previous_services,
simple_workflow_formatter))
else:
for r in RESPONSE_SELECTORS:
service, workers, session, gateway = \
make_service_from_config_rec(r, session,
StateManager.add_bot_utterance_simple_dict,
['RESPONSE_SELECTORS'], previous_services,
gateway)
services.append(service)
worker_tasks.extend(workers)
previous_services = {
i.name
for i in services if 'RESPONSE_SELECTORS' in i.tags
}
if POSTPROCESSORS:
for p in POSTPROCESSORS:
service, workers, session, gateway = make_service_from_config_rec(
p, session, StateManager.add_text_dict, ['POSTPROCESSORS'],
previous_services, gateway)
services.append(service)
worker_tasks.extend(workers)
previous_services = {
i.name
for i in services if 'POSTPROCESSORS' in i.tags
}
if ANNOTATORS_1:
for anno in ANNOTATORS_1:
service, workers, session, gateway = make_service_from_config_rec(
anno, session, StateManager.add_annotation_dict,
['POST_ANNOTATORS_1'], previous_services, gateway,
add_bot_to_name)
services.append(service)
worker_tasks.extend(workers)
previous_services = {
i.name
for i in services if 'POST_ANNOTATORS_1' in i.tags
}
if ANNOTATORS_2:
for anno in ANNOTATORS_2:
service, workers, session, gateway = make_service_from_config_rec(
anno, session, StateManager.add_annotation_dict,
['POST_ANNOTATORS_2'], previous_services, gateway,
add_bot_to_name)
services.append(service)
worker_tasks.extend(workers)
previous_services = {
i.name
for i in services if 'POST_ANNOTATORS_2' in i.tags
}
for anno in ANNOTATORS_3:
service, workers, session, gateway = make_service_from_config_rec(
anno, session, StateManager.add_annotation_dict,
['POST_ANNOTATORS_3'], previous_services, gateway, add_bot_to_name)
services.append(service)
worker_tasks.extend(workers)
return services, worker_tasks, session, gateway
class OsmocomBTS(BaseBTS):
REGISTERED_AUTH_VALUES = [
1,
] # 0 = camped, open reg. 1 = camped, auth'd
DEFAULT_BTS_ID = 0
DEFAULT_TRX_ID = 0
"""Osmocom services, order does matter. The dependency chart
looks like this:
osmo-trx ---\
-- osmo-bts-trx --\
osmocom-nitb / osmo-pcu -- openggsn -- osmo-sgsn
\
-- osmo-sip-connector -- mod_sofia (freeswitch)
-- mod_smpp (freeswitch)
"""
SERVICES = [
Service.SupervisorService('osmo-trx'),
Service.SystemService('osmocom-nitb'),
Service.SystemService('osmo-bts-trx'),
Service.SystemService('osmo-sip-connector'),
Service.SystemService('osmo-pcu'),
Service.SystemService('openggsn'),
Service.SystemService('osmo-sgsn')
]
def __init__(self):
self.conf = ConfigDB()
self.subscribers = Subscribers(
host=self.conf['bts.osmocom.ip'],
port=self.conf['bts.osmocom.bsc_vty_port'],
hlr_loc=self.conf['bts.osmocom.hlr_loc'],
timeout=self.conf['bss_timeout'])
self.network = Network(host=self.conf['bts.osmocom.ip'],
port=self.conf['bts.osmocom.bsc_vty_port'],
timeout=self.conf['bss_timeout'])
self.bts = BTS(host=self.conf['bts.osmocom.ip'],
port=self.conf['bts.osmocom.bsc_vty_port'],
timeout=self.conf['bss_timeout'])
self.trx = TRX(host=self.conf['bts.osmocom.ip'],
port=self.conf['bts.osmocom.bsc_vty_port'],
timeout=self.conf['bss_timeout'])
def set_factory_config(self):
pass
def get_camped_subscribers(self, access_period=0, auth=1):
try:
with self.subscribers as s:
return s.camped_subscribers(access_period, auth)
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_load(self):
try:
with self.bts as b:
load = b.get_load(self.DEFAULT_BTS_ID)
# If we weren't able to read any channel load
# the transceiver is not running
if sum(load.values()) == 0:
raise BSSError("TRX not running")
return load
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_noise(self):
return {'some_noise_stats_here_tbd': 0}
def set_mcc(self, mcc):
"""Set MCC"""
try:
with self.network as n:
return n.set_mcc(mcc)
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def set_mnc(self, mnc):
"""Set MNC"""
try:
with self.network as n:
return n.set_mnc(mnc)
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def set_short_name(self, short_name):
"""Set beacon short name"""
try:
with self.network as n:
return n.set_short_name(short_name)
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def set_open_registration(self, expression):
"""Set a regular expression matching IMSIs
that can camp to the network"""
raise NotImplementedError(
"Osmocom needs to implement this. Only has token auth for Ad-Hoc networks."
)
def set_timer(self, timer, value):
"""Set a particular BTS timer.
The only timer in use currently is T3212"""
try:
if str(timer) == '3212':
with self.bts as b:
return b.set_periodic_location_update(
self.DEFAULT_BTS_ID, value)
else:
with self.network as n:
return n.set_timer(timer, value)
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def set_band(self, band):
"""Set the GSM band of default BTS"""
try:
with self.bts as b:
return b.set_band(self.DEFAULT_BTS_ID, band)
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def set_arfcn_c0(self, arfcn):
"""Set the ARFCN of the first carrier."""
try:
with self.trx as t:
return t.set_arfcn(self.DEFAULT_BTS_ID, self.DEFAULT_TRX_ID,
arfcn)
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_mcc(self):
try:
with self.network as n:
return n.show()['mcc']
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_mnc(self):
try:
with self.network as n:
return n.show()['mnc']
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_short_name(self):
try:
with self.network as n:
return n.show()['short_name']
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_open_registration(self):
raise NotImplementedError()
def get_timer(self, timer):
try:
if str(timer) == '3212':
with self.bts as b:
return b.running_config(
self.DEFAULT_BTS_ID)['periodic location update']
else:
with self.network as n:
return n.running_config()['timer t%d' % timer]
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_available_bands(self):
try:
with self.bts as b:
return b.get_available_bands()
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_available_arfcns(self):
"""Returns a list of available ARFCNs for the default BTS"""
try:
with self.bts as b:
return b.get_available_arfcns(self.DEFAULT_BTS_ID)
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_band(self):
try:
with self.bts as b:
return b.show(self.DEFAULT_BTS_ID)['band']
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_arfcn_c0(self):
try:
with self.trx as t:
return t.show(self.DEFAULT_BTS_ID,
self.DEFAULT_TRX_ID)['arfcn']
except Exception as e:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError, "%s: %s" % (exc_type, exc_value), exc_trace
def get_gprs_usage(self, target_imsi=None):
"""Get all available GPRS data, or that of a specific IMSI (experimental).
Will return a dict of the form: {
'ipaddr': '192.168.99.1',
'downloaded_bytes': 200,
'uploaded_bytes': 100,
}
Or, if no IMSI is specified, multiple dicts like the one above will be
returned as part of a larger dict, keyed by IMSI.
Args:
target_imsi: the subsciber-of-interest
"""
raise NotImplementedError()
def get_versions(self):
#custom keys for this BTS type
versions = BaseBTS.get_versions(self)
versions['osmocom-public'] = self.conf['gsm_version']
versions['python-osmocom'] = self.conf['python-gsm_version']
return versions
def __init__(self, directory_name):
if os.path.exists(directory_name) == False:
os.mkdir(directory_name)
self.directory_name = directory_name
self.service = Service(directory_name)
import pigpio
import flask
import os
# initialize logging
logging.basicConfig(level=logging.DEBUG)
# initialize webserver
app = flask.Flask("logitech-z906")
# load GPIO
pi = pigpio.pi()
if not pi.connected:
logging.error("GPIO not available")
exit(0)
# initialize core
state = State()
queue = Queue()
# initialize components
service = Service(state, queue)
service.register(Api(pi, state, queue, app))
service.register(Controller(pi, state, queue))
service.register(Inputs(pi, state, queue))
service.register(Lirc(pi, state, queue))
service.register(Panel(pi, state, queue))
# run application
service.start()
def test_shared_connector(self):
"""Multiple instances of TestService share a Test service control."""
test_service = Service.SystemService("foo",
controller=TestServiceControl)
self.assertEqual(test_service.controller, self.service_ctl)
import delegator
import requests
from requests.exceptions import RequestException
from snowflake import snowflake
from ccm.common import logger
from core import system_utilities
from core.bts import bts
from core.config_database import ConfigDB
from core.servicecontrol import ServiceState
from core.service import Service
conf = ConfigDB()
# Dependent service names for start/stop.
SERVICES = bts.SERVICES + [
Service.SystemService('ccm-openvpn'),
Service.SystemService('freeswitch'),
Service.SystemService('endagad')
]
class RegistrationError(Exception):
def __init__(self, prefix, msg):
super(RegistrationError,
self).__init__('%s: %s' % (prefix, msg) if prefix else msg)
class RegistrationClientError(RegistrationError):
""" Exception was raised by client (could be socket, requests, etc.) """
def __init__(self, msg, ex, prefix=None):
super(RegistrationClientError, self).__init__(prefix,
class OpenBTSBTS(BaseBTS):
REGISTERED_AUTH_VALUES = [1, 2] # 0=reject, 1=open reg auth'd, 2=auth'd
SERVICES = [
Service.SupervisorService('openbts'),
Service.SupervisorService('smqueue'),
Service.SupervisorService('sipauthserve'),
Service.SupervisorService('gprsd')
]
def __init__(self):
self.conf = ConfigDB()
self.openbts = openbts.components.OpenBTS(
socket_timeout=self.conf['bss_timeout'],
cli_timeout=self.conf['bss_timeout'])
def restart(self):
"""An openbts specific command to restart the bts."""
delegator.run("sudo killall transceiver")
# If the pid file specified doesn't match a running instance of the
# process, remove the PID file. This is a workaround for a recurring
# OpenBTS issue we see. Note, caller must have permissions to remove file.
# Determine PIDs associated with pname
path = "/var/run/OpenBTS.pid"
cmd = delegator.run("ps -A | grep OpenBTS")
if cmd.return_code != 0:
output = ''
else:
output = cmd.out
pids = []
for line in output.split('\n'):
try:
pids.append(int(line.strip().split()[0]))
except ValueError:
continue # not a pid, so ignore it
except IndexError:
continue # malformed, ignore
try:
with open(path, "r") as f:
pid = int(f.read().strip())
if pid not in pids:
os.remove(path)
except IOError as e:
# ignore ENOENT (pid file doesn't exist)
if e.errno != errno.ENOENT:
raise
# now restart openbts
Service.SupervisorService("openbts").restart()
def set_factory_config(self):
"""
Verifies that OpenBTS TRX frequency offset settings are set to factory
defaults. This is only necessary for RAD1-based systems.
If not set to defaults, we need to update OpenBTS. First, we use the
`freqcorr` command to immediately update the offset; then, we set the
proper self.configuration value so we will use that in the future (as this is a
static variable, it's not enough to just update the self.config).
The default value is defined as follows. First, we check the self.configDB for a
"RealTRXFreqOffset" key. If this is defined, we use that. If this is
not defined, we assume the value listed as default for the
TRX.RadioFrequencyOffset key in OpenBTS is correct; this comes from the
trxfactory settings burned into the radio. Unfortunately, some radios from
Range are not properly calibrated, so we perform that calibration in-house
and set the RealTRXFreqOffset flag in our own self.config DB. We also don't have
a script implemented to update the contents of the RAD1 EEPROM, so easier
to just do this.
We also change the radio band if necessary -- there's some weirdness in the
implementation described below. If the band is changed, we must restart
OpenBTS for those changes to take effect, so we do that here as well.
Returns:
Whether or not OpenBTS needs to be restarted
TODO(shaddi): This does not support non-RAD1 systems, we need to add
support for UHD.
"""
restart_required = False
# First set frequency offset -- only needed for RAD1
try:
res = self.openbts.read_config("TRX.RadioFrequencyOffset")
except TimeoutError as e:
logger.error(
"Unable to query OpenBTS, can't set factory defaults!")
raise BSSError(e)
try:
default_offset = self.conf['RealTRXFreqOffset']
except KeyError:
default_offset = res.data['defaultValue']
if default_offset != res.data['value']:
self.openbts.update_config("TRX.RadioFrequencyOffset",
default_offset)
# We run this command via the CLI to immediately update the frequency
# offset.
r = delegator.run("/OpenBTS/OpenBTSCLI -c 'freqcorr %s'" %
(default_offset, ),
timeout=self.conf['bss_timeout'])
if r.return_code != 0:
err = "Error %s: %s" % (r.return_code, " ".join(r.cmd))
logger.error(err)
raise BSSError(err)
logger.notice("Frequency offset update to %s" % default_offset)
# Set band
res = self.openbts.read_config("GSM.Radio.Band")
if res.data['defaultValue'] != res.data['value']:
# We use delegator to update this value instead of the NodeManager
# interface because of a weird behavior in self.openbts. We rely on OpenBTS
# to report what ARFCNs in the band it supports when we read the self.config
# of GSM.Radio.C0 (specifically the validValues section). If we update
# the self.config via NodeManager, the GSM.Radio.Band self.config setting would
# be applied, but then reading the value of GSM.Radio.C0 will still
# return the previous band's set of valid values! For whatever reason,
# updating the band setting from the CLI will give us the valid ARFCNs
# for the new band setting.
logger.notice("Trying to set radio band from %s to %s" %
(res.data['value'], res.data['defaultValue']))
r = delegator.run(
"/OpenBTS/OpenBTSCLI -c 'config GSM.Radio.Band %s'" %
(res.data['defaultValue'], ),
timeout=self.conf['bss_timeout'])
if r.return_code != 0:
err = "Error %s: %s" % (r.return_code, " ".join(r.cmd))
logger.error(err)
raise BSSError(err)
restart_required = True
logger.notice("Updated radio band to %s" %
res.data['defaultValue'])
# Set ARFCN to lowest for the band
res = self.openbts.read_config("GSM.Radio.C0")
valid_arfcns = [_.split("|")[0] \
for _ in res.data['validValues'].split(",")]
if valid_arfcns[0] != res.data['value']:
logger.notice("Trying to set radio ARFCN from %s to %s" %
(res.data['value'], valid_arfcns[0]))
self.openbts.update_config("GSM.Radio.C0", valid_arfcns[0])
logger.notice("Updated ARFCN to %s" % valid_arfcns[0])
return True
return restart_required
def get_camped_subscribers(self, access_period=0, auth=1):
try:
return self.openbts.tmsis(access_period, auth)
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_load(self):
try:
tower_load = self.openbts.get_load()
return {
'sdcch_load':
tower_load['sdcch_load'],
'sdcch_available':
tower_load['sdcch_available'],
'tchf_load':
tower_load['tchf_load'],
'tchf_available':
tower_load['tchf_available'],
'pch_active':
tower_load['pch_active'],
'pch_total':
tower_load['pch_total'],
'agch_active':
tower_load['agch_active'],
'agch_pending':
tower_load['agch_pending'],
'gprs_current_pdchs':
tower_load['gprs_current_pdchs'],
'gprs_utilization_percentage':
(tower_load['gprs_utilization_percentage']),
}
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_noise(self):
try:
tower_noise = self.openbts.get_noise()
return {
'noise_rssi_db':
tower_noise['noise_rssi_db'],
'noise_ms_rssi_target_db':
(tower_noise['noise_ms_rssi_target_db']),
}
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def set_mcc(self, mcc):
"""Set MCC"""
try:
return self.openbts.update_config("GSM.Identity.MCC", mcc)
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def set_mnc(self, mnc):
"""Set MNC"""
try:
return self.openbts.update_config("GSM.Identity.MNC", mnc)
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def set_short_name(self, short_name):
"""Set beacon short name"""
try:
return self.openbts.update_config("GSM.Identity.ShortName",
short_name)
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def set_open_registration(self, expression):
"""Set a regular expression matching IMSIs
that can camp to the network"""
try:
return self.openbts.update_config("Control.LUR.OpenRegistration",
expression)
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def set_timer(self, timer, value):
"""Set a particular BTS timer.
The only timer in use currently is T3212"""
try:
return self.openbts.update_config("GSM.Timer.T%s" % str(timer),
value)
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def set_band(self, band):
"""Set the GSM band"""
try:
return self.openbts.update_config("GSM.Radio.Band", band)
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def set_arfcn_c0(self, arfcn):
"""Set the ARFCN of the first carrier."""
try:
return self.openbts.update_config("GSM.Radio.C0", arfcn)
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_mcc(self):
try:
res = self.openbts.read_config("GSM.Identity.MCC")
return res.data['value']
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_mnc(self):
try:
res = self.openbts.read_config("GSM.Identity.MNC")
return res.data['value']
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_short_name(self):
try:
res = self.openbts.read_config("GSM.Identity.ShortName")
return res.data['value']
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_open_registration(self):
try:
res = self.openbts.read_config("Control.LUR.OpenRegistration")
return res.data['value']
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_timer(self, timer):
try:
res = self.openbts.read_config("GSM.Timer.T%s" % str(timer))
return res.data['value']
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_available_bands(self):
try:
res = self.openbts.read_config("GSM.Radio.Band")
return res.data['validValues'].split(",")
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_available_arfcns(self):
try:
res = self.openbts.read_config("GSM.Radio.C0")
return res.data['validValues'].split(",")
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_band(self):
# as per github.com/endaga/openbts/GSM/GSMConfig.cpp:gsmInit()
# valid keys are: 850,900,1800,1900
# let's convert those to CCM standard names (e.g., GSM900)
try:
res = self.openbts.read_config("GSM.Radio.Band")
#convert to CCM standard
return "GSM" + str(res.data['value'])
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_arfcn_c0(self):
try:
res = self.openbts.read_config("GSM.Radio.C0")
return res.data['value']
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
raise BSSError("%s: %s" %
(exc_type, exc_value)).with_traceback(exc_trace)
def get_versions(self):
#custom keys for this BTS type
versions = BaseBTS.get_versions(self)
versions['openbts-public'] = self.conf['gsm_version']
versions['python-openbts'] = self.conf['python-gsm_version'] #hack
return versions
def setUp(self) -> None:
"""Instantiate the service."""
self.service = Service()
import requests
from requests.exceptions import RequestException
from snowflake import snowflake
from ccm.common import logger
from core import system_utilities
from core.bts import bts
from core.exceptions import BSSError
from core.config_database import ConfigDB
from core.servicecontrol import ServiceState
from core.service import Service
conf = ConfigDB()
# Dependent supervisor service names for start/stop.
SERVICES = bts.SERVICES + [Service.SupervisorService('openvpn'),
Service.SystemService('freeswitch'),
Service.SupervisorService('endagad')]
class RegistrationError(Exception):
def __init__(self, prefix, msg):
super(RegistrationError, self).__init__(
'%s: %s' % (prefix, msg) if prefix else msg)
class RegistrationClientError(RegistrationError):
""" Exception was raised by client (could be socket, requests, etc.) """
def __init__(self, msg, ex, prefix=None):
super(RegistrationClientError, self).__init__(
prefix, msg + (': %s' % (ex, )))
def run_default():
services, workers, session, gateway = parse_old_config()
if CHANNEL == 'cmd_client':
endpoint = Service('cmd_responder',
EventSetOutputConnector('cmd_responder').send,
StateManager.save_dialog_dict, 1, ['responder'])
input_srv = Service('input', None,
StateManager.add_human_utterance_simple_dict, 1,
['input'])
loop = asyncio.get_event_loop()
loop.set_debug(args.debug)
register_msg, process = prepare_agent(
services,
endpoint,
input_srv,
use_response_logger=args.response_logger)
if gateway:
gateway.on_channel_callback = register_msg
gateway.on_service_callback = process
future = asyncio.ensure_future(run(register_msg))
for i in workers:
loop.create_task(i.call_service(process))
try:
loop.run_until_complete(future)
except KeyboardInterrupt:
pass
except Exception as e:
raise e
finally:
future.cancel()
if session:
loop.run_until_complete(session.close())
if gateway:
gateway.disconnect()
loop.stop()
loop.close()
logging.shutdown()
elif CHANNEL == 'http_client':
if not session:
session = ClientSession()
intermediate_storage = {}
endpoint = Service(
'http_responder',
HttpOutputConnector(intermediate_storage, 'http_responder').send,
StateManager.save_dialog_dict, 1, ['responder'])
input_srv = Service('input', None,
StateManager.add_human_utterance_simple_dict, 1,
['input'])
register_msg, process_callable = prepare_agent(services, endpoint,
input_srv,
args.response_logger)
if gateway:
gateway.on_channel_callback = register_msg
gateway.on_service_callback = process_callable
app = init_app(register_msg, intermediate_storage,
prepare_startup(workers, process_callable, session),
on_shutdown, args.debug)
web.run_app(app, port=args.port)
elif CHANNEL == 'telegram':
token = getenv('TELEGRAM_TOKEN')
proxy = getenv('TELEGRAM_PROXY')
loop = asyncio.get_event_loop()
bot = Bot(token=token, loop=loop, proxy=proxy)
dp = Dispatcher(bot)
endpoint = Service('telegram_responder',
EventSetOutputConnector('telegram_responder').send,
StateManager.save_dialog_dict, 1, ['responder'])
input_srv = Service('input', None,
StateManager.add_human_utterance_simple_dict, 1,
['input'])
register_msg, process = prepare_agent(
services,
endpoint,
input_srv,
use_response_logger=args.response_logger)
if gateway:
gateway.on_channel_callback = register_msg
gateway.on_service_callback = process
for i in workers:
loop.create_task(i.call_service(process))
tg_msg_processor = TelegramMessageProcessor(register_msg)
dp.message_handler()(tg_msg_processor.handle_message)
executor.start_polling(dp, skip_updates=True)
def setUpClass(cls):
cls.test_service = Service.SystemService("foo",
controller=TestServiceControl)
cls.service_ctl = cls.test_service.controller
# faq = build_model(configs.faq.tfidf_autofaq, download=True)
# sentiment = build_model(configs.classifiers.rusentiment_elmo_twitter_rnn, download=True)
# utterances = ['Привет!', 'Когда началась Вторая Мировая?',
# 'Привет, я бот!', '1939', 'Как дела?', 'Спасибо, бот!',
# 'Хорошо, а у тебя как?', 'И у меня нормально. Когда родился Петр Первый?',
# 'в 1672 году', 'спасибо', ]
# print("DeepPavlov configs output:")
# print(ner(utterances))
# print(faq(utterances))
# print(sentiment(utterances))
state_manager = StateManager()
anno_names, anno_urls = zip(*[(annotator['name'], annotator['url'])
for annotator in ANNOTATORS])
preprocessor = Service(rest_caller=RestCaller(
max_workers=MAX_WORKERS, names=anno_names, urls=anno_urls))
postprocessor = DefaultPostprocessor()
skill_caller = RestCaller(max_workers=MAX_WORKERS)
response_selector = ConfidenceResponseSelector()
ss_names, ss_urls = zip(*[(selector['name'], selector['url'])
for selector in SKILL_SELECTORS])
skill_selector = ChitchatQASelector(
RestCaller(max_workers=MAX_WORKERS, names=ss_names, urls=ss_urls))
skill_manager = SkillManager(skill_selector=skill_selector,
response_selector=response_selector,
skill_caller=skill_caller)
agent = Agent(state_manager, preprocessor, postprocessor, skill_manager)
# TEST predict_annotations()
def run(self):
"""
Main loop for endagad. This moves the system through the various
states of operation -- it should be a state machine really!
General flow is:
1) Tries to get configuration from server to produce VPN keys
2) Generates keys locally.
3) Sends CSR for signing, returns that.
4) Starts system services (FS, BTS, etc) and configures them
appropriately. Note configuration can change depending on registration
and VPN state of the system.
5) Runs checkin periodically.
"""
eapi = interconnect.endaga_ic(self._conf)
if 'registration_interval' not in self._conf:
self._conf['registration_interval'] = 60
UNHEALTHY_THRESH = self._conf.get('bts.unhealthy_threshold', 3)
unhealthy_count = UNHEALTHY_THRESH # fail quickly on first pass
while True:
# Retrieve keys/tokens, or do nothing if we have them.
logger.notice("Performing gen_keys")
registration.generate_keys()
# generate_keys() loads auth token on success. Need to update the
# interconnect client's token if so.
if eapi.token is None:
eapi.token = self._conf['endaga_token']
# Try to register/get VPN credentials. Tries forever if fails.
logger.notice("Performing register")
registration.register(eapi)
# Registered, start services and tries to start VPN. Stop
# everything otherwise.
logger.notice("Performing clear_pid")
registration.clear_old_pid()
logger.notice("Performing update_vpn")
registration.update_vpn()
# At this point, all services should be up, so we can perform
# additional configuration.
self._reset_bts_config()
# Update the inbound_url if the VPN is up.
if system_utilities.get_vpn_ip() is not None:
logger.notice("Performing register_update")
registration.register_update(eapi)
logger.notice("Performing ensure_fs_external_bound")
registration.ensure_fs_external_bound_to_vpn()
# Send checkin to cloud
try:
# Sends events, tries to get config info. Can proceed w/o VPN.
logger.notice("Performing checkin.")
checkin_data = eapi.checkin(timeout=30)
logger.notice("Performing system health check.")
if not registration.system_healthcheck(checkin_data):
unhealthy_count += 1
logger.notice("System unhealthy: %d" % unhealthy_count)
else:
unhealthy_count = 0
except (ConnectionError, Timeout):
logger.error(
"checkin failed due to connection error or timeout.")
except BSSError as e:
logger.error("bts exception: %s" % e)
if unhealthy_count > UNHEALTHY_THRESH:
logger.notice("BTS seems unhealthy, restarting BTS services.")
bts.restart()
Service.SystemService("freeswitch").restart()
# Upgrade the endaga metapackage, when appropriate and only if that
# feature is enabled.
logger.notice("Performing autoupgrade")
system_utilities.try_to_autoupgrade()
# Sleep for some amount of time before retrying
logger.notice("Performing sleep")
time.sleep(self._conf['registration_interval'])
import requests
import requests.exceptions
import snowflake
from ccm.common import logger
from core import system_utilities
from core.bts import bts
from core.exceptions import BSSError
from core.config_database import ConfigDB
from core.servicecontrol import ServiceState
from core.service import Service
conf = ConfigDB()
# Dependent supervisor service names for start/stop.
SERVICES = bts.SERVICES + [
Service.SupervisorService('openvpn'),
Service.SystemService('freeswitch'),
Service.SupervisorService('endagad')
]
def get_registration_conf():
"""Attempts to get registration config information from the cloud.
Returns:
the config data
Raises:
ValueError if the request does not return 200
"""
params = {'bts_uuid': snowflake.snowflake()}
