def make_app(global_conf, **app_conf):
"""Returns a Sync Server Application."""
global_conf.update(app_conf)
params = Config(global_conf)
app = klass(urls, controllers, params, auth_class)
if params.get('debug', False):
app = TransLogger(app, logger_name='syncserver',
setup_console_handler=True)
if params.get('profile', False):
from repoze.profile.profiler import AccumulatingProfileMiddleware
app = AccumulatingProfileMiddleware(app,
log_filename='profile.log',
cachegrind_filename='cachegrind.out',
discard_first_request=True,
flush_at_shutdown=True,
path='/__profile__')
if params.get('client_debug', False):
# errors are displayed in the user client
app = ErrorMiddleware(app, debug=True,
show_exceptions_in_wsgi_errors=True)
else:
# errors are logged and a 500 is returned with an empty body
# to avoid any security whole
app = CatchErrorMiddleware(app, logger_name='syncserver')
if wrapper is not None:
app = wrapper(app, config=params)
return app
def make_app(global_conf, **app_conf):
"""Returns a Key Exchange Application."""
global_conf.update(app_conf)
config = Config(global_conf)
app = KeyExchangeApp(config)
blacklisted = app.blacklisted
# hooking a profiler
if global_conf.get('profile', 'false').lower() == 'true':
from repoze.profile.profiler import AccumulatingProfileMiddleware
app = AccumulatingProfileMiddleware(app, log_filename='profile.log',
cachegrind_filename='cachegrind.out',
discard_first_request=True,
flush_at_shutdown=True,
path='/__profile__')
# hooking a client debugger
if global_conf.get('client_debug', 'false').lower() == 'true':
from paste.exceptions.errormiddleware import ErrorMiddleware
app = ErrorMiddleware(app, debug=True,
show_exceptions_in_wsgi_errors=True)
# hooking a stdout logger
if global_conf.get('debug', 'false').lower() == 'true':
from paste.translogger import TransLogger
app = TransLogger(app, logger_name='jpakeapp',
setup_console_handler=True)
# IP Filtering middleware
if config.get('filtering.use', False):
del config['filtering.use']
params = config.get_section('filtering')
app = IPFiltering(app, callback=blacklisted, **params)
return app
def make_app(global_conf, **app_conf):
"""Returns a Sync Server Application."""
global_conf.update(app_conf)
params = Config(global_conf)
app = klass(urls, controllers, params, auth_class)
if params.get('debug', False):
app = TransLogger(app,
logger_name='syncserver',
setup_console_handler=True)
if params.get('profile', False):
from repoze.profile.profiler import AccumulatingProfileMiddleware
app = AccumulatingProfileMiddleware(
app,
log_filename='profile.log',
cachegrind_filename='cachegrind.out',
discard_first_request=True,
flush_at_shutdown=True,
path='/__profile__')
if params.get('client_debug', False):
# errors are displayed in the user client
app = ErrorMiddleware(app,
debug=True,
show_exceptions_in_wsgi_errors=True)
else:
# errors are logged and a 500 is returned with an empty body
# to avoid any security whole
app = CatchErrorMiddleware(app, logger_name='syncserver')
if wrapper is not None:
app = wrapper(app, config=params)
return app
def make_sse_server(config_filename):
configItems = ['configuration',
'broker_host',
'broker_amqp_port',
'broker_username',
'broker_password',
'broker_virtual_host',
'incoming_exchange_name',
'notifs_queue_name']
SSEConfig = Config(config_filename)
configMap = dict([(item, SSEConfig.get("app:post_office", item))
for item in configItems])
return ServerEventController(configMap)
def setUp(self):
test_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'tests.conf')
self.config = Config(cfgfile = test_cfg)
self.app = TestApp(make_app(self.config))
self.app.reset()
def _init(cls):
cls.log = logging.getLogger(__name__)
cls.log.info("Initializing databases")
config = Config().get_configuration()
cls.db = PostgresqlDatabase(config.get("RESOURCES_DB", "name"),
user=config.get("RESOURCES_DB", "user"),
password=config.get(
"RESOURCES_DB", "password"),
host=config.get("RESOURCES_DB", "host"))
cls._influx_token = config.get("INFLUX_DB", "token")
cls.influx_org = config.get("INFLUX_DB", "org")
cls.influx_bucket = config.get("INFLUX_DB", "bucket")
cls.influx_client = InfluxDBClient(url=config.get(
"INFLUX_DB", "server"),
token=cls._influx_token)
cls.influx_write_api = cls.influx_client.write_api(
write_options=SYNCHRONOUS)
def test_config(self):
cfg_in = {'one': '1', 'two': 'bla', 'three': 'false'}
config = Config(cfg_in)
self.assertTrue(config['one'])
self.assertEqual(config['two'], 'bla')
self.assertFalse(config['three'])
# config also reads extra config files.
__, filename = tempfile.mkstemp()
try:
with open(filename, 'w') as f:
f.write(_EXTRA)
cfg_in = {'one': '1', 'two': 'file:%s' % filename}
config.load_config(cfg_in)
self.assertTrue(config['some.stuff'])
self.assertEquals(config['other.thing'], 'ok')
finally:
os.remove(filename)
def test_config(self):
cfg_in = {'one': '1', 'two': 'bla', 'three': 'false'}
config = Config(cfg_in)
self.assertTrue(config['one'])
self.assertEqual(config['two'], 'bla')
self.assertFalse(config['three'])
# config also reads extra config files.
__, filename = tempfile.mkstemp()
try:
with open(filename, 'w') as f:
f.write(_EXTRA)
cfg_in = {'one': '1', 'two': 'file:%s' % filename}
config.load_config(cfg_in)
self.assertTrue(config['some.stuff'])
self.assertEquals(config['other.thing'], 'ok')
finally:
os.remove(filename)
def convert_config(config):
"""Loads the configuration.
If a "configuration" option is found, reads it using config.Config.
Each section/option is then converted to "section.option" in the resulting
mapping.
"""
res = {}
for key, value in config.items():
if not isinstance(value, basestring) or not value.startswith('file:'):
res[key] = convert(value)
continue
# we load the configuration and inject it in the mapping
filename = value[len('file:'):]
if not os.path.exists(filename):
raise ValueError('The configuration file was not found. "%s"' % \
filename)
conf = Config(filename)
res.update(conf.get_map())
return res
def convert_config(config):
"""Loads the configuration.
If a "configuration" option is found, reads it using config.Config.
Each section/option is then converted to "section.option" in the resulting
mapping.
"""
res = {}
for key, value in config.items():
if not isinstance(value, basestring) or not value.startswith('file:'):
res[key] = convert(value)
continue
# we load the configuration and inject it in the mapping
filename = value[len('file:'):]
if not os.path.exists(filename):
raise ValueError('The configuration file was not found. "%s"' % \
filename)
conf = Config(filename)
res.update(conf.get_map())
return res
def test_reader(self):
config = Config(self.file_one)
# values conversion
self.assertEquals(config.get('one', 'foo'), 'bar')
self.assertEquals(config.get('one', 'num'), -12)
self.assertEquals(config.get('one', 'st'), 'o=k')
self.assertEquals(config.get('one', 'lines'), [1, 'two', 3])
self.assertEquals(config.get('one', 'env'), 'some stuff')
# getting a map
map = config.get_map()
self.assertEquals(map['one.foo'], 'bar')
map = config.get_map('one')
self.assertEquals(map['foo'], 'bar')
del os.environ['__STUFF__']
self.assertRaises(EnvironmentNotFoundError, config.get, 'one', 'env')
# extends
self.assertEquals(config.get('three', 'more'), 'stuff')
self.assertEquals(config.get('one', 'two'), 'a')
def test_load_direct(self):
from services.tests.test_pluginreg import Dummy
bad_config = Config({'backend': 'xxx'})
good_config = Config({'test.backend':
'services.tests.test_pluginreg.Dummy',
'test.foo': 'bar'})
self.assertRaises(KeyError, load_and_configure, bad_config)
obj = load_and_configure(good_config, 'test')
self.assertTrue(isinstance(obj, Dummy))
self.assertTrue(obj.foo == 'bar')
missing_interface = {'interface': 'services.tests.test_pluginreg.Nope'}
bad_interface = {'backend':
'services.tests.test_pluginreg.ImplementsBadly',
'interface':
'services.tests.test_pluginreg.ClassInterface'}
good_interface = {'backend':
'services.tests.test_pluginreg.ImplementsCorrectly',
'interface':
'services.tests.test_pluginreg.ClassInterface'}
obj = load_and_configure(good_interface)
self.assertTrue(isinstance(obj, ImplementsCorrectly))
self.assertRaises(TypeError, load_and_configure, bad_interface)
self.assertRaises(ImportError, load_and_configure, missing_interface)
def setUp(self):
if not MEMCACHED:
raise SkipTest
# Ensure we have metlog loaded so the timers will work.
config_file = os.path.join(os.path.dirname(__file__), "sync.conf")
config = Config(cfgfile=config_file)
load_and_configure(config, "metlog_loader")
fd, self.dbfile = mkstemp()
os.close(fd)
self.fn = 'syncstorage.storage.memcachedsql.MemcachedSQLStorage'
kwds = self.STORAGE_CONFIG.copy()
kwds['sqluri'] = 'sqlite:///%s' % self.dbfile
self.storage = SyncStorage.get(self.fn, **kwds)
# make sure we have the standard collections in place
for name in ('client', 'crypto', 'forms', 'history'):
self.storage.set_collection(_UID, name)
class ClientAgentTest(unittest.TestCase):
"""Test class for all Client Agent tests.
This handles the creation of an in-process Client Agent server to run
tests against.
"""
def setUp(self):
test_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests.conf")
self.config = Config(cfgfile=test_cfg)
# self.appdir = env.topdir
self.app = TestApp(make_app(self.config))
self.app.reset()
def tearDown(self):
pass
def set_credentials(self, username=None, password=None):
self.username = username
if password is None:
jws = JWS()
password = jws.sign(
header={"alg": "NONE", "typ": "IDAssertion"},
payload={"exp": int(time.time() + 300), "moz-vep-id": username},
)
self.password = password
def create_queue(self):
res = self.app.post("/%s/new_queue" % VERSION, extra_environ={"test_session.uid": self.username})
assert res.status_int == 200
return json.loads(res.body)
def create_subscription(self, token):
res = self.app.post(
"/%s/new_subscription" % VERSION,
json.dumps({"token": token}),
content_type="application/json",
extra_environ={"test_session.uid": self.username},
)
assert res.status_int == 200
def remove_subscription(self, token, success_response=200):
res = self.app.post(
"/%s/remove_subscription" % VERSION,
json.dumps({"token": token}),
status=success_response,
content_type="application/json",
extra_environ={"test_session.uid": self.username},
)
assert res.status_int == success_response
def send_broadcast(self, message=None):
if message is None:
message = "'this is a broadcast message'"
self.app.post(
"/%s/broadcast" % (VERSION),
message,
content_type="application/json",
extra_environ={"test_session.uid": self.username},
)
# TODO: Figure out a way to extract messages from queues in tests
# For now we assume all is well if "200 OK" returned
def test_create_queue(self):
self.set_credentials(self.config.get("tests.user", "*****@*****.**"), self.config.get("tests.password", None))
queue_info = self.create_queue()
assert "queue_id" in queue_info
assert "host" in queue_info
assert "port" in queue_info
assert len(queue_info["queue_id"]) > 0
assert len(queue_info["host"]) > 0
assert queue_info["port"]
def test_subscriptions(self):
self.set_credentials(self.config.get("tests.user", "*****@*****.**"), self.config.get("tests.password", None))
token = "TEST123"
# Can't delete subscription if it doesn't exist
try:
self.remove_subscription(token, success_response=400)
except Exception, e:
print str(e)
# Normal create and delete
self.create_subscription(token)
self.remove_subscription(token)
# Can't delete subscription if already deleted
self.remove_subscription(token, success_response=400)
def setUp(self):
test_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests.conf")
self.config = Config(cfgfile=test_cfg)
# self.appdir = env.topdir
self.app = TestApp(make_app(self.config))
self.app.reset()
def start_processes(config_ini):
procs = []
stoplist = []
logger.info("Starting config service")
conf = Config()
proc = Process(target=conf.run, args=(config_ini,))
procs.append(proc)
proc.start()
if wait_service_up(SOCK_CONFIG) is False:
stop_services(stoplist)
sys.exit(1)
stoplist.append(SOCK_CONFIG)
logger.info("Starting service-detection service")
service_detection = ServiceDetection()
proc = Process(target=service_detection.run, args=())
procs.append(proc)
proc.start()
if wait_service_up(SOCK_SD) is False:
stop_services(stoplist)
sys.exit(1)
stoplist.append(SOCK_SD)
logger.info("Starting DNS service")
dns = DNSAPI()
proc = Process(target=dns.run, args=())
procs.append(proc)
proc.start()
if wait_service_up(SOCK_DNS) is False:
stop_services(stoplist)
sys.exit(1)
stoplist.append(SOCK_DNS)
logger.info("Starting web-server service")
mgmt_server = ManageWebservers()
proc = Process(target=mgmt_server.run, args=())
procs.append(proc)
proc.start()
if wait_service_up(SOCK_WEBSERVER) is False:
stop_services(stoplist)
sys.exit(1)
stoplist.append(SOCK_WEBSERVER)
logger.info("Starting module-loader service")
module_ldr = ModuleLoader()
proc = Process(target=module_ldr.run, args=())
procs.append(proc)
proc.start()
if wait_service_up(SOCK_MODULES) is False:
stop_services(stoplist)
sys.exit(1)
stoplist.append(SOCK_MODULES)
logger.info("Starting database service")
database = Database()
proc = Process(target=database.run, args=())
procs.append(proc)
proc.start()
if wait_service_up(SOCK_DATABASE) is False:
stop_services(stoplist)
sys.exit(1)
stoplist.append(SOCK_DATABASE)
logger.info("Starting shell-receiver service")
shell = ShellReceiver()
proc = Process(target=shell.run, args=())
procs.append(proc)
proc.start()
# No need to wait for shell-service to be up
logger.info("All processes have started")
return procs
def ajax_config_setconfig(rd, user, data):
filterOperation('setConfig', user, data.attr)
Config.SetValue(data.attr, data.data)
def test_config_merge(self):
config = Config(cfgfile=self.file_four)
global_ = {'foo': 'bar', 'baz': 'bawlp'}
self.assertEqual(config.get_section(''), dict())
self.assertEqual(config.get_section('global'), global_)
self.assertEqual(config['auth.a'], 'b')
self.assertEqual(config['auth.c'], 'd')
self.assertEqual(config.get_section('auth'), {'a': 'b', 'c': 'd'})
storage = {'e': 'f', 'g': 'h'}
self.assertEqual(config['storage.e'], 'f')
self.assertEqual(config['storage.g'], 'h')
self.assertEqual(config.get_section('storage'), storage)
storage_once = {'i': 'j', 'k': 'l'}
storage_once.update(storage)
once_merged = config.merge('multi:once')
self.assertEqual(once_merged.get_section('storage'), storage_once)
storage_thrice = {'i': 'jjj', 'k': 'lll'}
storage_thrice.update(storage)
thrice_merged = config.merge('multi:thrice')
self.assertEqual(thrice_merged.get_section('storage'), storage_thrice)
# merge cache should guarantee same result
config['storage.m'] = 'n'
config['multi:thrice.storage.o'] = 'ppp'
thrice_merged = config.merge('multi:thrice')
self.assertEqual(thrice_merged.get_section('storage'), storage_thrice)
# but not after merge cache is cleared
config.clear_merge_cache()
thrice_merged = config.merge('multi:thrice')
self.assertNotEqual(thrice_merged.get_section('storage'),
storage_thrice)
storage_thrice.update(dict(m='n', o='ppp'))
self.assertEqual(thrice_merged.get_section('storage'), storage_thrice)
def test_config_merge(self):
config = Config(cfgfile=self.file_four)
global_ = {'foo': 'bar', 'baz': 'bawlp'}
self.assertEqual(config.get_section(''), dict())
self.assertEqual(config.get_section('global'), global_)
self.assertEqual(config['auth.a'], 'b')
self.assertEqual(config['auth.c'], 'd')
self.assertEqual(config.get_section('auth'),
{'a': 'b', 'c': 'd'})
storage = {'e': 'f', 'g': 'h'}
self.assertEqual(config['storage.e'], 'f')
self.assertEqual(config['storage.g'], 'h')
self.assertEqual(config.get_section('storage'),
storage)
storage_once = {'i': 'j', 'k': 'l'}
storage_once.update(storage)
once_merged = config.merge('multi:once')
self.assertEqual(once_merged.get_section('storage'),
storage_once)
storage_thrice = {'i': 'jjj', 'k': 'lll'}
storage_thrice.update(storage)
thrice_merged = config.merge('multi:thrice')
self.assertEqual(thrice_merged.get_section('storage'),
storage_thrice)
# merge cache should guarantee same result
config['storage.m'] = 'n'
config['multi:thrice.storage.o'] = 'ppp'
thrice_merged = config.merge('multi:thrice')
self.assertEqual(thrice_merged.get_section('storage'),
storage_thrice)
# but not after merge cache is cleared
config.clear_merge_cache()
thrice_merged = config.merge('multi:thrice')
self.assertNotEqual(thrice_merged.get_section('storage'),
storage_thrice)
storage_thrice.update(dict(m='n', o='ppp'))
self.assertEqual(thrice_merged.get_section('storage'),
storage_thrice)
def __init__(self, urls, controllers, config=None,
auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self)) for name, klass in
controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None) and
(not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None, match, controller=controller,
action=action, conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
def __init__(self, urls, controllers, config=None,
auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
self.modules = dict()
for module in self.config.get('app.modules', []):
self.modules[module] = load_and_configure(self.config, module)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self)) for name, klass in
controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
self.mapper.connect(None, match, controller=controller,
action=action, conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
def convert_config(self, ini_cfg, ini_path):
here = {'here': os.path.dirname(os.path.realpath(ini_path))}
cfg_in = dict([(key, value % here)
for key, value in ini_cfg.items('DEFAULT') +
ini_cfg.items('app:main')])
return Config(cfg_in)
class SyncServerApp(object):
""" Dispatches the request to the right controller by using Routes.
"""
def __init__(self, urls, controllers, config=None, auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self))
for name, klass in controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None)
and (not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None,
match,
controller=controller,
action=action,
conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
def _sigterm(self, signal, frame):
self.shutting = True
# wait for a bit
sleep(self.graceful_shutdown_interval)
# no more queries
self.killing = True
# wait for a bit
sleep(self.hard_shutdown_interval)
# now we can notify the end -- so pending stuff can be cleaned up
notify(APP_ENDS)
# bye-bye
sys.exit(0)
def _before_call(self, request):
return {}
def _host_specific(self, host, config):
"""Will compute host-specific requests"""
return config.merge('host:%s' % host)
#
# Debug & heartbeat pages
#
def _debug_server(self, request):
return []
def _check_server(self, request):
pass
def _debug(self, request):
return self.standard_controller._debug(request)
def _heartbeat(self, request):
return self.standard_controller._heartbeat(request)
# events fired when a request is handled
def _notified(func):
def __notified(self, request):
notify(REQUEST_STARTS, request)
response = None
try:
response = func(self, request)
return response
finally:
notify(REQUEST_ENDS, response)
return __notified
# information dumped in error logs
def get_infos(self, request):
"""Returns a mapping containing useful info.
It can be related to the request, or global to the app.
"""
return {'user': str(request.user)}
#
# entry point
#
@wsgify
@_notified
def __call__(self, request):
"""Entry point for the WSGI app."""
# the app is being killed, no more requests please
if self.killing:
raise HTTPServiceUnavailable()
request.server_time = round_time()
# gets request-specific config
request.config = self._host_specific(request.host, self.config)
# pre-hook
before_headers = self._before_call(request)
try:
response = self._dispatch_request(request)
except HTTPException, response:
# set before-call headers on all responses
response.headers.update(before_headers)
raise
else:
from services.bilibili_client import BilibiliClient
from services.config import Config
from os import linesep
content = ""
config = Config()
config.print()
bilibili = BilibiliClient(config)
reward = bilibili.get_reward()
print(reward)
avs = bilibili.get_regions(rid=20, num=5)
print(avs)
share = bilibili.share(avs[0]['aid'])
print(share)
def add_content(line):
global content
content = content.__add__(line).__add__(linesep)
reward_after = bilibili.get_reward()
add_content(
f"投币获得经验：{reward_after['coins_av']}，分享视频{'成功' if reward_after['share_av'] else '失败'}!!!"
)
add_content(
f"当前等级：{reward_after['level_info']['current_level']}，当前经验：{reward_after['level_info']['current_exp']}"
)
print(content)
class TestStorage(unittest.TestCase):
def setUp(self):
test_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'tests.conf')
self.config = Config(cfgfile = test_cfg)
self.app = TestApp(make_app(self.config))
self.app.reset()
def tearDown(self):
pass
def test_all(self):
storage = get_message_backend(self.config)
username = self.config.get('tests.user', '*****@*****.**')
test_token = 'TEST123'
origin = 'example.com'
test_message = json.dumps({"body": json.dumps({
"token": test_token,
"timestamp": int(time.time()),
"ciphertext": "test",
"ttl": 100}),
"HMAC": "123abc"})
queue_info = storage.create_client_queue(username)
subs = storage.create_subscription(username, test_token)
#Send a message to the user.
storage.send_broadcast(test_message,
username,
origin = origin,
queue = test_token)
msgs = storage.get_pending_messages(username)
self.assertEqual(json.loads(msgs[0].get('body')).get('token'),
test_token)
storage._purge(username = username)
#Send a message to a user based on their queue id (most common path)
storage.publish_message(test_message,
queue_info.get('queue_id'),
origin = origin)
msgs = storage.get_pending_messages(username)
self.failUnless(test_token in msgs[0].get('body'))
storage._purge(username = username)
# Add a message to a subscription queue
storage.queue_message(test_message,
subs.get('queue_id'),
origin = origin)
msgs = storage.get_pending_messages(username)
# clean all the messages out of the user queue
storage._purge(username = username)
# drop the subscription.
storage.delete_subscription(username, test_token)
# Should not be able to add message to a deleted queue
self.failIf(storage.queue_message(test_message,
subs.get('queue_id'),
origin = origin))
# Should not be able to read messages from an empty queue
msgs = storage.get_pending_messages(username)
self.failIf(len(msgs))
def init_config(logger):
from services.config import Config
config = Config().get_configuration()
logger.info("Cluster ID: " + config.get("CLUSTER", "id"))
class SyncServerApp(object):
""" Dispatches the request to the right controller by using Routes.
"""
def __init__(self, urls, controllers, config=None,
auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self)) for name, klass in
controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None) and
(not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None, match, controller=controller,
action=action, conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
def _sigterm(self, signal, frame):
self.shutting = True
# wait for a bit
sleep(self.graceful_shutdown_interval)
# no more queries
self.killing = True
# wait for a bit
sleep(self.hard_shutdown_interval)
# now we can notify the end -- so pending stuff can be cleaned up
notify(APP_ENDS)
# bye-bye
sys.exit(0)
def _before_call(self, request):
return {}
def _host_specific(self, host, config):
"""Will compute host-specific requests"""
return config.merge('host:%s' % host)
#
# Debug & heartbeat pages
#
def _debug_server(self, request):
return []
def _check_server(self, request):
pass
def _debug(self, request):
return self.standard_controller._debug(request)
def _heartbeat(self, request):
return self.standard_controller._heartbeat(request)
# events fired when a request is handled
def _notified(func):
def __notified(self, request):
notify(REQUEST_STARTS, request)
response = None
try:
response = func(self, request)
return response
finally:
notify(REQUEST_ENDS, response)
return __notified
# information dumped in error logs
def get_infos(self, request):
"""Returns a mapping containing useful info.
It can be related to the request, or global to the app.
"""
return {'user': str(request.user)}
#
# entry point
#
@wsgify
@_notified
def __call__(self, request):
"""Entry point for the WSGI app."""
# the app is being killed, no more requests please
if self.killing:
raise HTTPServiceUnavailable()
request.server_time = round_time()
# gets request-specific config
request.config = self._host_specific(request.host, self.config)
# pre-hook
before_headers = self._before_call(request)
try:
response = self._dispatch_request(request)
except HTTPException, response:
# set before-call headers on all responses
response.headers.update(before_headers)
raise
else:
def ajax_config_listconfig(rd, user, data):
filterOperation('listConfig', user)
return makeResponseSuccess(Config.ListAttrs())
def __init__(self, urls, controllers, config=None, auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self))
for name, klass in controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None)
and (not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None,
match,
controller=controller,
action=action,
conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
self.delivery_channel.basic_publish(
exchange=self.incoming_exchange_name,
routing_key=token,
body=message
)
print "Notification routed to user exchange"
except Exception as ex:
# TODO: Either put the message back in the incoming queue, or send
# a NACK to the broker if we're going to do ACK/NACK crap
raise ex
if __name__ == '__main__':
# Load configuration settings
config = Config(os.getcwd() + '/etc/notifs.conf')
config_map = config.get_map()
print "Starting routing server..."
router = RouterServer(config_map)
print "Routing server started."
try:
router.start()
except KeyboardInterrupt:
router.shutdown()
print "Routing server shut down."
import json, argparse, os, sys, importlib
import gym
from services.environments.minigrid_test import *
from services.util import load_class, load_object, load_model, load_policy, load_algorithm
from services.arguments import Arguments
from services.config import Config
from services.constants import *
pwd = os.path.dirname(os.path.realpath(__file__))
PARSER = Arguments(pwd=pwd).parser
PARSER.add_argument("--test_name")
if __name__ == "__main__":
args = PARSER.parse_args()
config = Config(pwd=pwd, args=args).config
env = gym.make(config.get(ENVIRONMENT_NAME, args.environment_name),
**config)
environment_wrapper_config = config.get(ENVIRONTMENT_WRAPPER)
if environment_wrapper_config:
for idx, module in enumerate(environment_wrapper_config[MODULES]
): # for ability to wrap mulitple
environment_wrapper = load_class(
module, environment_wrapper_config[CLASSES][idx])
env = environment_wrapper(env)
env.reset()
model = load_model(config[MODEL_MODULE])(environment=env, **config).model
policy = load_policy(config[POLICY_MODULE])(environment=env)
def ajax_config_getconfig(rd, user, data):
filterOperation('getConfig', user, data.attr)
return makeResponseSuccess(Config.__getattr__(data.attr))