def make_app(global_conf, **app_conf):
"""Returns a Sync Server Application."""
global_conf.update(app_conf)
params = Config(global_conf)
app = klass(urls, controllers, params, auth_class)
if params.get('debug', False):
app = TransLogger(app, logger_name='syncserver',
setup_console_handler=True)
if params.get('profile', False):
from repoze.profile.profiler import AccumulatingProfileMiddleware
app = AccumulatingProfileMiddleware(app,
log_filename='profile.log',
cachegrind_filename='cachegrind.out',
discard_first_request=True,
flush_at_shutdown=True,
path='/__profile__')
if params.get('client_debug', False):
# errors are displayed in the user client
app = ErrorMiddleware(app, debug=True,
show_exceptions_in_wsgi_errors=True)
else:
# errors are logged and a 500 is returned with an empty body
# to avoid any security whole
app = CatchErrorMiddleware(app, logger_name='syncserver')
if wrapper is not None:
app = wrapper(app, config=params)
return app
def make_app(global_conf, **app_conf):
"""Returns a Sync Server Application."""
global_conf.update(app_conf)
params = Config(global_conf)
app = klass(urls, controllers, params, auth_class)
if params.get('debug', False):
app = TransLogger(app,
logger_name='syncserver',
setup_console_handler=True)
if params.get('profile', False):
from repoze.profile.profiler import AccumulatingProfileMiddleware
app = AccumulatingProfileMiddleware(
app,
log_filename='profile.log',
cachegrind_filename='cachegrind.out',
discard_first_request=True,
flush_at_shutdown=True,
path='/__profile__')
if params.get('client_debug', False):
# errors are displayed in the user client
app = ErrorMiddleware(app,
debug=True,
show_exceptions_in_wsgi_errors=True)
else:
# errors are logged and a 500 is returned with an empty body
# to avoid any security whole
app = CatchErrorMiddleware(app, logger_name='syncserver')
if wrapper is not None:
app = wrapper(app, config=params)
return app
def make_app(global_conf, **app_conf):
"""Returns a Key Exchange Application."""
global_conf.update(app_conf)
config = Config(global_conf)
app = KeyExchangeApp(config)
blacklisted = app.blacklisted
# hooking a profiler
if global_conf.get('profile', 'false').lower() == 'true':
from repoze.profile.profiler import AccumulatingProfileMiddleware
app = AccumulatingProfileMiddleware(app, log_filename='profile.log',
cachegrind_filename='cachegrind.out',
discard_first_request=True,
flush_at_shutdown=True,
path='/__profile__')
# hooking a client debugger
if global_conf.get('client_debug', 'false').lower() == 'true':
from paste.exceptions.errormiddleware import ErrorMiddleware
app = ErrorMiddleware(app, debug=True,
show_exceptions_in_wsgi_errors=True)
# hooking a stdout logger
if global_conf.get('debug', 'false').lower() == 'true':
from paste.translogger import TransLogger
app = TransLogger(app, logger_name='jpakeapp',
setup_console_handler=True)
# IP Filtering middleware
if config.get('filtering.use', False):
del config['filtering.use']
params = config.get_section('filtering')
app = IPFiltering(app, callback=blacklisted, **params)
return app
def make_sse_server(config_filename):
configItems = ['configuration',
'broker_host',
'broker_amqp_port',
'broker_username',
'broker_password',
'broker_virtual_host',
'incoming_exchange_name',
'notifs_queue_name']
SSEConfig = Config(config_filename)
configMap = dict([(item, SSEConfig.get("app:post_office", item))
for item in configItems])
return ServerEventController(configMap)
def _init(cls):
cls.log = logging.getLogger(__name__)
cls.log.info("Initializing databases")
config = Config().get_configuration()
cls.db = PostgresqlDatabase(config.get("RESOURCES_DB", "name"),
user=config.get("RESOURCES_DB", "user"),
password=config.get(
"RESOURCES_DB", "password"),
host=config.get("RESOURCES_DB", "host"))
cls._influx_token = config.get("INFLUX_DB", "token")
cls.influx_org = config.get("INFLUX_DB", "org")
cls.influx_bucket = config.get("INFLUX_DB", "bucket")
cls.influx_client = InfluxDBClient(url=config.get(
"INFLUX_DB", "server"),
token=cls._influx_token)
cls.influx_write_api = cls.influx_client.write_api(
write_options=SYNCHRONOUS)
def test_reader(self):
config = Config(self.file_one)
# values conversion
self.assertEquals(config.get('one', 'foo'), 'bar')
self.assertEquals(config.get('one', 'num'), -12)
self.assertEquals(config.get('one', 'st'), 'o=k')
self.assertEquals(config.get('one', 'lines'), [1, 'two', 3])
self.assertEquals(config.get('one', 'env'), 'some stuff')
# getting a map
map = config.get_map()
self.assertEquals(map['one.foo'], 'bar')
map = config.get_map('one')
self.assertEquals(map['foo'], 'bar')
del os.environ['__STUFF__']
self.assertRaises(EnvironmentNotFoundError, config.get, 'one', 'env')
# extends
self.assertEquals(config.get('three', 'more'), 'stuff')
self.assertEquals(config.get('one', 'two'), 'a')
import gym
from services.environments.minigrid_test import *
from services.util import load_class, load_object, load_model, load_policy, load_algorithm
from services.arguments import Arguments
from services.config import Config
from services.constants import *
pwd = os.path.dirname(os.path.realpath(__file__))
PARSER = Arguments(pwd=pwd).parser
PARSER.add_argument("--test_name")
if __name__ == "__main__":
args = PARSER.parse_args()
config = Config(pwd=pwd, args=args).config
env = gym.make(config.get(ENVIRONMENT_NAME, args.environment_name),
**config)
environment_wrapper_config = config.get(ENVIRONTMENT_WRAPPER)
if environment_wrapper_config:
for idx, module in enumerate(environment_wrapper_config[MODULES]
): # for ability to wrap mulitple
environment_wrapper = load_class(
module, environment_wrapper_config[CLASSES][idx])
env = environment_wrapper(env)
env.reset()
model = load_model(config[MODEL_MODULE])(environment=env, **config).model
policy = load_policy(config[POLICY_MODULE])(environment=env)
algorithm = load_algorithm(config[ALGORITHM_MODULE])(environment=env,
model=model,
class SyncServerApp(object):
""" Dispatches the request to the right controller by using Routes.
"""
def __init__(self, urls, controllers, config=None,
auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self)) for name, klass in
controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None) and
(not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None, match, controller=controller,
action=action, conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
def _sigterm(self, signal, frame):
self.shutting = True
# wait for a bit
sleep(self.graceful_shutdown_interval)
# no more queries
self.killing = True
# wait for a bit
sleep(self.hard_shutdown_interval)
# now we can notify the end -- so pending stuff can be cleaned up
notify(APP_ENDS)
# bye-bye
sys.exit(0)
def _before_call(self, request):
return {}
def _host_specific(self, host, config):
"""Will compute host-specific requests"""
return config.merge('host:%s' % host)
#
# Debug & heartbeat pages
#
def _debug_server(self, request):
return []
def _check_server(self, request):
pass
def _debug(self, request):
return self.standard_controller._debug(request)
def _heartbeat(self, request):
return self.standard_controller._heartbeat(request)
# events fired when a request is handled
def _notified(func):
def __notified(self, request):
notify(REQUEST_STARTS, request)
response = None
try:
response = func(self, request)
return response
finally:
notify(REQUEST_ENDS, response)
return __notified
# information dumped in error logs
def get_infos(self, request):
"""Returns a mapping containing useful info.
It can be related to the request, or global to the app.
"""
return {'user': str(request.user)}
#
# entry point
#
@wsgify
@_notified
def __call__(self, request):
"""Entry point for the WSGI app."""
# the app is being killed, no more requests please
if self.killing:
raise HTTPServiceUnavailable()
request.server_time = round_time()
# gets request-specific config
request.config = self._host_specific(request.host, self.config)
# pre-hook
before_headers = self._before_call(request)
try:
response = self._dispatch_request(request)
except HTTPException, response:
# set before-call headers on all responses
response.headers.update(before_headers)
raise
else:
class ClientAgentTest(unittest.TestCase):
"""Test class for all Client Agent tests.
This handles the creation of an in-process Client Agent server to run
tests against.
"""
def setUp(self):
test_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests.conf")
self.config = Config(cfgfile=test_cfg)
# self.appdir = env.topdir
self.app = TestApp(make_app(self.config))
self.app.reset()
def tearDown(self):
pass
def set_credentials(self, username=None, password=None):
self.username = username
if password is None:
jws = JWS()
password = jws.sign(
header={"alg": "NONE", "typ": "IDAssertion"},
payload={"exp": int(time.time() + 300), "moz-vep-id": username},
)
self.password = password
def create_queue(self):
res = self.app.post("/%s/new_queue" % VERSION, extra_environ={"test_session.uid": self.username})
assert res.status_int == 200
return json.loads(res.body)
def create_subscription(self, token):
res = self.app.post(
"/%s/new_subscription" % VERSION,
json.dumps({"token": token}),
content_type="application/json",
extra_environ={"test_session.uid": self.username},
)
assert res.status_int == 200
def remove_subscription(self, token, success_response=200):
res = self.app.post(
"/%s/remove_subscription" % VERSION,
json.dumps({"token": token}),
status=success_response,
content_type="application/json",
extra_environ={"test_session.uid": self.username},
)
assert res.status_int == success_response
def send_broadcast(self, message=None):
if message is None:
message = "'this is a broadcast message'"
self.app.post(
"/%s/broadcast" % (VERSION),
message,
content_type="application/json",
extra_environ={"test_session.uid": self.username},
)
# TODO: Figure out a way to extract messages from queues in tests
# For now we assume all is well if "200 OK" returned
def test_create_queue(self):
self.set_credentials(self.config.get("tests.user", "*****@*****.**"), self.config.get("tests.password", None))
queue_info = self.create_queue()
assert "queue_id" in queue_info
assert "host" in queue_info
assert "port" in queue_info
assert len(queue_info["queue_id"]) > 0
assert len(queue_info["host"]) > 0
assert queue_info["port"]
def test_subscriptions(self):
self.set_credentials(self.config.get("tests.user", "*****@*****.**"), self.config.get("tests.password", None))
token = "TEST123"
# Can't delete subscription if it doesn't exist
try:
self.remove_subscription(token, success_response=400)
except Exception, e:
print str(e)
# Normal create and delete
self.create_subscription(token)
self.remove_subscription(token)
# Can't delete subscription if already deleted
self.remove_subscription(token, success_response=400)
class TestStorage(unittest.TestCase):
def setUp(self):
test_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'tests.conf')
self.config = Config(cfgfile = test_cfg)
self.app = TestApp(make_app(self.config))
self.app.reset()
def tearDown(self):
pass
def test_all(self):
storage = get_message_backend(self.config)
username = self.config.get('tests.user', '*****@*****.**')
test_token = 'TEST123'
origin = 'example.com'
test_message = json.dumps({"body": json.dumps({
"token": test_token,
"timestamp": int(time.time()),
"ciphertext": "test",
"ttl": 100}),
"HMAC": "123abc"})
queue_info = storage.create_client_queue(username)
subs = storage.create_subscription(username, test_token)
#Send a message to the user.
storage.send_broadcast(test_message,
username,
origin = origin,
queue = test_token)
msgs = storage.get_pending_messages(username)
self.assertEqual(json.loads(msgs[0].get('body')).get('token'),
test_token)
storage._purge(username = username)
#Send a message to a user based on their queue id (most common path)
storage.publish_message(test_message,
queue_info.get('queue_id'),
origin = origin)
msgs = storage.get_pending_messages(username)
self.failUnless(test_token in msgs[0].get('body'))
storage._purge(username = username)
# Add a message to a subscription queue
storage.queue_message(test_message,
subs.get('queue_id'),
origin = origin)
msgs = storage.get_pending_messages(username)
# clean all the messages out of the user queue
storage._purge(username = username)
# drop the subscription.
storage.delete_subscription(username, test_token)
# Should not be able to add message to a deleted queue
self.failIf(storage.queue_message(test_message,
subs.get('queue_id'),
origin = origin))
# Should not be able to read messages from an empty queue
msgs = storage.get_pending_messages(username)
self.failIf(len(msgs))
class SyncServerApp(object):
""" Dispatches the request to the right controller by using Routes.
"""
def __init__(self, urls, controllers, config=None, auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self))
for name, klass in controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None)
and (not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None,
match,
controller=controller,
action=action,
conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
def _sigterm(self, signal, frame):
self.shutting = True
# wait for a bit
sleep(self.graceful_shutdown_interval)
# no more queries
self.killing = True
# wait for a bit
sleep(self.hard_shutdown_interval)
# now we can notify the end -- so pending stuff can be cleaned up
notify(APP_ENDS)
# bye-bye
sys.exit(0)
def _before_call(self, request):
return {}
def _host_specific(self, host, config):
"""Will compute host-specific requests"""
return config.merge('host:%s' % host)
#
# Debug & heartbeat pages
#
def _debug_server(self, request):
return []
def _check_server(self, request):
pass
def _debug(self, request):
return self.standard_controller._debug(request)
def _heartbeat(self, request):
return self.standard_controller._heartbeat(request)
# events fired when a request is handled
def _notified(func):
def __notified(self, request):
notify(REQUEST_STARTS, request)
response = None
try:
response = func(self, request)
return response
finally:
notify(REQUEST_ENDS, response)
return __notified
# information dumped in error logs
def get_infos(self, request):
"""Returns a mapping containing useful info.
It can be related to the request, or global to the app.
"""
return {'user': str(request.user)}
#
# entry point
#
@wsgify
@_notified
def __call__(self, request):
"""Entry point for the WSGI app."""
# the app is being killed, no more requests please
if self.killing:
raise HTTPServiceUnavailable()
request.server_time = round_time()
# gets request-specific config
request.config = self._host_specific(request.host, self.config)
# pre-hook
before_headers = self._before_call(request)
try:
response = self._dispatch_request(request)
except HTTPException, response:
# set before-call headers on all responses
response.headers.update(before_headers)
raise
else: