def test_obfuscation(self):
req = make_request("/__debug__", _ENVIRON)
controller = StandardController(None)
def _more_secret(*args):
return ['stuff', 'and', 'pymysql://*****:*****@localhost/sync']
controller._debug_server = _more_secret
debug = controller._debug(req)
# make sure we don't have any password left
self.assertTrue('xxxx' not in debug.body)
def test_obfuscation(self):
req = make_request("/__debug__", _ENVIRON)
controller = StandardController(None)
def _more_secret(*args):
return ['stuff', 'and', 'pymysql://*****:*****@localhost/sync']
controller._debug_server = _more_secret
debug = controller._debug(req)
# make sure we don't have any password left
self.assertTrue('xxxx' not in debug.body)
def __init__(self, urls, controllers, config=None,
auth_class=Authentication):
self.mapper = Mapper()
if config is not None:
self.config = config
else:
self.config = {}
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self)) for name, klass in
controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
self.mapper.connect(None, match, controller=controller,
action=action, conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
def __init__(self, urls, controllers, config=None,
auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self)) for name, klass in
controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None) and
(not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None, match, controller=controller,
action=action, conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
class SyncServerApp(object):
""" Dispatches the request to the right controller by using Routes.
"""
def __init__(self, urls, controllers, config=None,
auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self)) for name, klass in
controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None) and
(not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None, match, controller=controller,
action=action, conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
def _sigterm(self, signal, frame):
self.shutting = True
# wait for a bit
sleep(self.graceful_shutdown_interval)
# no more queries
self.killing = True
# wait for a bit
sleep(self.hard_shutdown_interval)
# now we can notify the end -- so pending stuff can be cleaned up
notify(APP_ENDS)
# bye-bye
sys.exit(0)
def _before_call(self, request):
return {}
def _host_specific(self, host, config):
"""Will compute host-specific requests"""
return config.merge('host:%s' % host)
#
# Debug & heartbeat pages
#
def _debug_server(self, request):
return []
def _check_server(self, request):
pass
def _debug(self, request):
return self.standard_controller._debug(request)
def _heartbeat(self, request):
return self.standard_controller._heartbeat(request)
# events fired when a request is handled
def _notified(func):
def __notified(self, request):
notify(REQUEST_STARTS, request)
response = None
try:
response = func(self, request)
return response
finally:
notify(REQUEST_ENDS, response)
return __notified
# information dumped in error logs
def get_infos(self, request):
"""Returns a mapping containing useful info.
It can be related to the request, or global to the app.
"""
return {'user': str(request.user)}
#
# entry point
#
@wsgify
@_notified
def __call__(self, request):
"""Entry point for the WSGI app."""
# the app is being killed, no more requests please
if self.killing:
raise HTTPServiceUnavailable()
request.server_time = round_time()
# gets request-specific config
request.config = self._host_specific(request.host, self.config)
# pre-hook
before_headers = self._before_call(request)
try:
response = self._dispatch_request(request)
except HTTPException, response:
# set before-call headers on all responses
response.headers.update(before_headers)
raise
else:
class SyncServerApp(object):
""" Dispatches the request to the right controller by using Routes.
"""
def __init__(self, urls, controllers, config=None,
auth_class=Authentication):
self.mapper = Mapper()
if config is not None:
self.config = config
else:
self.config = {}
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self)) for name, klass in
controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
self.mapper.connect(None, match, controller=controller,
action=action, conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
def _before_call(self, request):
return {}
def _host_specific(self, host, config):
"""Will compute host-specific requests"""
if host in self._host_configs:
return self._host_configs[host]
# overrides the original value with the host-specific value
host_section = 'host:%s.' % host
host_config = {}
overriden_keys = []
for key, value in config.items():
if key in overriden_keys:
continue
if key.startswith(host_section):
key = key[len(host_section):]
overriden_keys.append(key)
host_config[key] = value
self._host_configs[host] = host_config
return host_config
#
# Debug & heartbeat pages
#
def _debug_server(self, request):
return []
def _check_server(self, request):
pass
def _debug(self, request):
return self.standard_controller._debug(request)
def _heartbeat(self, request):
return self.standard_controller._heartbeat(request)
#
# entry point
#
@wsgify
def __call__(self, request):
if request.method in ('HEAD',):
raise HTTPBadRequest('"%s" not supported' % request.method)
request.server_time = round_time()
# gets request-specific config
request.config = self._host_specific(request.host, self.config)
# pre-hook
before_headers = self._before_call(request)
# XXX
# removing the trailing slash - ambiguity on client side
url = request.path_info.rstrip('/')
if url != '':
request.environ['PATH_INFO'] = request.path_info = url
if (self.heartbeat_page is not None and
url == '/%s' % self.heartbeat_page):
return self._heartbeat(request)
if self.debug_page is not None and url == '/%s' % self.debug_page:
return self._debug(request)
match = self.mapper.routematch(environ=request.environ)
if match is None:
return HTTPNotFound()
match, __ = match
# authentication control
if self.auth is not None:
self.auth.check(request, match)
function = self._get_function(match['controller'], match['action'])
if function is None:
raise HTTPNotFound('Unkown URL %r' % request.path_info)
# extracting all the info from the headers and the url
request.sync_info = match
# the GET mapping is filled on GET and DELETE requests
if request.method in ('GET', 'DELETE'):
params = dict(request.GET)
else:
params = {}
try:
result = function(request, **params)
except BackendError:
err = traceback.format_exc()
logger.error(err)
raise HTTPServiceUnavailable(retry_after=self.retry_after)
if isinstance(result, basestring):
response = getattr(request, 'response', None)
if response is None:
response = Response(result)
elif isinstance(result, str):
response.body = result
else:
# if it's not str it's unicode, which really shouldn't happen
module = getattr(function, '__module__', 'unknown')
name = getattr(function, '__name__', 'unknown')
logger.warn('Unicode response returned from: %s - %s'
% (module, name))
response.unicode_body = result
else:
# result is already a Response
response = result
# setting up the X-Weave-Timestamp
response.headers['X-Weave-Timestamp'] = str(request.server_time)
response.headers.update(before_headers)
return response
def _get_function(self, controller, action):
"""Return the action of the right controller."""
try:
controller = self.controllers[controller]
except KeyError:
return None
return getattr(controller, action, None)
def __init__(self, urls, controllers, config=None,
auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
self.modules = dict()
for module in self.config.get('app.modules', []):
self.modules[module] = load_and_configure(self.config, module)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self)) for name, klass in
controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
self.mapper.connect(None, match, controller=controller,
action=action, conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
def __init__(self, urls, controllers, config=None, auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self))
for name, klass in controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None)
and (not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None,
match,
controller=controller,
action=action,
conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
class SyncServerApp(object):
""" Dispatches the request to the right controller by using Routes.
"""
def __init__(self, urls, controllers, config=None, auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self))
for name, klass in controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None)
and (not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None,
match,
controller=controller,
action=action,
conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
def _sigterm(self, signal, frame):
self.shutting = True
# wait for a bit
sleep(self.graceful_shutdown_interval)
# no more queries
self.killing = True
# wait for a bit
sleep(self.hard_shutdown_interval)
# now we can notify the end -- so pending stuff can be cleaned up
notify(APP_ENDS)
# bye-bye
sys.exit(0)
def _before_call(self, request):
return {}
def _host_specific(self, host, config):
"""Will compute host-specific requests"""
return config.merge('host:%s' % host)
#
# Debug & heartbeat pages
#
def _debug_server(self, request):
return []
def _check_server(self, request):
pass
def _debug(self, request):
return self.standard_controller._debug(request)
def _heartbeat(self, request):
return self.standard_controller._heartbeat(request)
# events fired when a request is handled
def _notified(func):
def __notified(self, request):
notify(REQUEST_STARTS, request)
response = None
try:
response = func(self, request)
return response
finally:
notify(REQUEST_ENDS, response)
return __notified
# information dumped in error logs
def get_infos(self, request):
"""Returns a mapping containing useful info.
It can be related to the request, or global to the app.
"""
return {'user': str(request.user)}
#
# entry point
#
@wsgify
@_notified
def __call__(self, request):
"""Entry point for the WSGI app."""
# the app is being killed, no more requests please
if self.killing:
raise HTTPServiceUnavailable()
request.server_time = round_time()
# gets request-specific config
request.config = self._host_specific(request.host, self.config)
# pre-hook
before_headers = self._before_call(request)
try:
response = self._dispatch_request(request)
except HTTPException, response:
# set before-call headers on all responses
response.headers.update(before_headers)
raise
else:
