def __init__(self, conf, db):
"""
Initialize a LimitContainer. This sets up an appropriate
control daemon, as well as providing a container for the
limits themselves.
:param conf: A turnstile.config.Config instance containing the
configuration for the ControlDaemon.
:param db: A database handle for the Redis database.
"""
self.conf = conf
self.db = db
self.limits = []
self.limit_map = {}
self.limit_sum = None
# Initialize the control daemon
if conf.to_bool(conf['control'].get('remote', 'no'), False):
self.control_daemon = remote.RemoteControlDaemon(self, conf)
else:
self.control_daemon = control.ControlDaemon(self, conf)
# Now start the control daemon
self.control_daemon.start()
def test_reload_exception_altkeys(self, mock_format_exc, mock_exception):
cd = control.ControlDaemon(
'middleware',
config.Config(
conf_dict={
'control.errors_key': 'alt_err',
'control.errors_channel': 'alt_chan',
}))
cd.pending = mock.Mock(**{'acquire.return_value': True})
cd.limits = mock.Mock(
**{
'set_limits.side_effect': test_utils.TestException,
})
cd._db = mock.Mock(**{'zrange.return_value': ['limit1', 'limit2']})
cd.reload()
cd.pending.assert_has_calls([
mock.call.acquire(False),
mock.call.release(),
])
self.assertEqual(len(cd.pending.method_calls), 2)
cd.limits.set_limits.assert_called_once_with(['limit1', 'limit2'])
cd._db.assert_has_calls([
mock.call.zrange('limits', 0, -1),
mock.call.sadd('alt_err', 'Failed to load limits: <traceback>'),
mock.call.publish('alt_chan',
'Failed to load limits: <traceback>'),
])
self.assertEqual(len(cd._db.method_calls), 3)
mock_exception.assert_called_once_with('Could not load limits')
mock_format_exc.assert_called_once_with()
def test_start(self, mock_reload, mock_spawn_n):
cd = control.ControlDaemon('middleware', 'config')
cd.start()
mock_spawn_n.assert_called_once_with(cd.listen)
self.assertEqual(cd.listen_thread, 'listen_thread')
mock_reload.assert_called_once_with()
def test_init(self):
cd = control.ControlDaemon('middleware', 'config')
self.assertEqual(cd._db, None)
self.assertEqual(cd.middleware, 'middleware')
self.assertEqual(cd.config, 'config')
self.assertIsInstance(cd.limits, control.LimitData)
self.assertIsInstance(cd.pending, eventlet.semaphore.Semaphore)
self.assertEqual(cd.listen_thread, None)
def test_reload_noacquire(self, mock_format_exc, mock_exception):
cd = control.ControlDaemon('middleware', config.Config())
cd.pending = mock.Mock(**{'acquire.return_value': False})
cd.limits = mock.Mock()
cd._db = mock.Mock()
cd.reload()
cd.pending.assert_has_calls([
mock.call.acquire(False),
])
self.assertEqual(len(cd.pending.method_calls), 1)
self.assertEqual(len(cd.limits.method_calls), 0)
self.assertEqual(len(cd._db.method_calls), 0)
self.assertFalse(mock_exception.called)
self.assertFalse(mock_format_exc.called)
def test_listen_shardhint(self, mock_get_database):
pubsub = mock.Mock(**{'listen.return_value': []})
db = mock.Mock(**{'pubsub.return_value': pubsub})
mock_get_database.return_value = db
cd = control.ControlDaemon(
'middleware',
config.Config(conf_dict={
'control.shard_hint': 'shard',
}))
cd.listen()
mock_get_database.assert_called_once_with('control')
db.pubsub.assert_called_once_with(shard_hint='shard')
pubsub.assert_has_calls([
mock.call.subscribe('control'),
mock.call.listen(),
])
def test_reload(self, mock_format_exc, mock_exception):
cd = control.ControlDaemon('middleware', config.Config())
cd.pending = mock.Mock(**{'acquire.return_value': True})
cd.limits = mock.Mock()
cd._db = mock.Mock(**{'zrange.return_value': ['limit1', 'limit2']})
cd.reload()
cd.pending.assert_has_calls([
mock.call.acquire(False),
mock.call.release(),
])
self.assertEqual(len(cd.pending.method_calls), 2)
cd.limits.set_limits.assert_called_once_with(['limit1', 'limit2'])
cd._db.assert_has_calls([
mock.call.zrange('limits', 0, -1),
])
self.assertEqual(len(cd._db.method_calls), 1)
self.assertFalse(mock_exception.called)
self.assertFalse(mock_format_exc.called)
def test_db_middleware(self):
middleware = mock.Mock(db='midware_db')
cd = control.ControlDaemon(middleware, config.Config())
self.assertEqual(cd.db, 'midware_db')
def test_db_present(self):
middleware = mock.Mock(db='midware_db')
cd = control.ControlDaemon(middleware, config.Config())
cd._db = 'cached_db'
self.assertEqual(cd.db, 'cached_db')
def test_get_limits(self):
cd = control.ControlDaemon('middleware', 'config')
cd.limits = 'limits'
self.assertEqual(cd.get_limits(), 'limits')
def test_listen_altchan(self, mock_exception, mock_error,
mock_get_database, mock_find_entrypoint):
pubsub = mock.Mock(
**{
'listen.return_value': [
{
'type': 'other',
'channel': 'control',
'data': 'ping',
},
{
'type': 'pmessage',
'channel': 'other',
'data': 'ping',
},
{
'type': 'message',
'channel': 'other',
'data': 'ping',
},
{
'type': 'pmessage',
'channel': 'control',
'data': '',
},
{
'type': 'message',
'channel': 'control',
'data': '',
},
{
'type': 'pmessage',
'channel': 'control',
'data': '_ping',
},
{
'type': 'message',
'channel': 'control',
'data': '_ping',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'nosuch',
},
{
'type': 'message',
'channel': 'control',
'data': 'nosuch',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'fail',
},
{
'type': 'message',
'channel': 'control',
'data': 'fail',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'fail:arg1:arg2',
},
{
'type': 'message',
'channel': 'control',
'data': 'fail:arg1:arg2',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'ping',
},
{
'type': 'message',
'channel': 'control',
'data': 'ping',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'ping:arg1:arg2',
},
{
'type': 'message',
'channel': 'control',
'data': 'ping:arg1:arg2',
},
]
})
db = mock.Mock(**{'pubsub.return_value': pubsub})
mock_get_database.return_value = db
cd = control.ControlDaemon(
'middleware',
config.Config(conf_dict={
'control.channel': 'other',
}))
cd.listen()
mock_get_database.assert_called_once_with('control')
db.pubsub.assert_called_once_with()
pubsub.assert_has_calls([
mock.call.subscribe('other'),
mock.call.listen(),
])
self.assertFalse(mock_error.called)
self.assertFalse(mock_exception.called)
control.ControlDaemon._commands['ping'].assert_has_calls([
mock.call(cd),
mock.call(cd),
])
self.assertFalse(control.ControlDaemon._commands['_ping'].called)
self.assertFalse(control.ControlDaemon._commands['fail'].called)
self.assertFalse(mock_find_entrypoint.called)
def test_listen(self, mock_exception, mock_error, mock_get_database,
mock_find_entrypoint):
entrypoints = dict(discovered=mock.Mock())
mock_find_entrypoint.side_effect = (
lambda x, y, compat: entrypoints.get(y))
pubsub = mock.Mock(
**{
'listen.return_value': [
{
'type': 'other',
'channel': 'control',
'data': 'ping',
},
{
'type': 'pmessage',
'channel': 'other',
'data': 'ping',
},
{
'type': 'message',
'channel': 'other',
'data': 'ping',
},
{
'type': 'pmessage',
'channel': 'control',
'data': '',
},
{
'type': 'message',
'channel': 'control',
'data': '',
},
{
'type': 'pmessage',
'channel': 'control',
'data': '_ping',
},
{
'type': 'message',
'channel': 'control',
'data': '_ping',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'nosuch',
},
{
'type': 'message',
'channel': 'control',
'data': 'nosuch',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'fail',
},
{
'type': 'message',
'channel': 'control',
'data': 'fail',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'fail:arg1:arg2',
},
{
'type': 'message',
'channel': 'control',
'data': 'fail:arg1:arg2',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'ping',
},
{
'type': 'message',
'channel': 'control',
'data': 'ping',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'ping:arg1:arg2',
},
{
'type': 'message',
'channel': 'control',
'data': 'ping:arg1:arg2',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'discovered',
},
{
'type': 'message',
'channel': 'control',
'data': 'discovered',
},
{
'type': 'pmessage',
'channel': 'control',
'data': 'discovered:arg1:arg2',
},
{
'type': 'message',
'channel': 'control',
'data': 'discovered:arg1:arg2',
},
]
})
db = mock.Mock(**{'pubsub.return_value': pubsub})
mock_get_database.return_value = db
cd = control.ControlDaemon('middleware', config.Config())
cd.listen()
mock_get_database.assert_called_once_with('control')
db.pubsub.assert_called_once_with()
pubsub.assert_has_calls([
mock.call.subscribe('control'),
mock.call.listen(),
])
mock_error.assert_has_calls([
mock.call("Cannot call internal command '_ping'"),
mock.call("Cannot call internal command '_ping'"),
mock.call("No such command 'nosuch'"),
mock.call("No such command 'nosuch'"),
])
mock_exception.assert_has_calls([
mock.call("Failed to execute command 'fail' arguments []"),
mock.call("Failed to execute command 'fail' arguments []"),
mock.call("Failed to execute command 'fail' arguments "
"['arg1', 'arg2']"),
mock.call("Failed to execute command 'fail' arguments "
"['arg1', 'arg2']"),
])
control.ControlDaemon._commands['ping'].assert_has_calls([
mock.call(cd),
mock.call(cd),
mock.call(cd, 'arg1', 'arg2'),
mock.call(cd, 'arg1', 'arg2'),
])
self.assertFalse(control.ControlDaemon._commands['_ping'].called)
control.ControlDaemon._commands['fail'].assert_has_calls([
mock.call(cd),
mock.call(cd),
mock.call(cd, 'arg1', 'arg2'),
mock.call(cd, 'arg1', 'arg2'),
])
entrypoints['discovered'].assert_has_calls([
mock.call(cd),
mock.call(cd),
mock.call(cd, 'arg1', 'arg2'),
mock.call(cd, 'arg1', 'arg2'),
])
mock_find_entrypoint.assert_has_calls([
mock.call('turnstile.command', 'nosuch', compat=False),
mock.call('turnstile.command', 'discovered', compat=False),
])
self.assertEqual(len(mock_find_entrypoint.mock_calls), 2)
self.assertEqual(control.ControlDaemon._commands['discovered'],
entrypoints['discovered'])
self.assertEqual(control.ControlDaemon._commands['nosuch'], None)
def __init__(self, app, local_conf):
"""
Initialize the turnstile middleware. Saves the configuration
and sets up the list of preprocessors, connects to the
database, and initiates the control daemon thread.
"""
# Save the application
self.app = app
self.limits = []
self.limit_sum = None
self.mapper = None
self.mapper_lock = eventlet.semaphore.Semaphore()
# Save the configuration
self.conf = config.Config(conf_dict=local_conf)
# We will lazy-load the database
self._db = None
# Set up request pre- and post-processors
self.preprocessors = []
self.postprocessors = []
enable = self.conf.get('enable')
if enable is not None:
# Use the enabler syntax
for proc in enable.split():
# Try the preprocessor
preproc = utils.find_entrypoint('turnstile.preprocessor',
proc,
compat=False)
if preproc:
self.preprocessors.append(preproc)
# Now the postprocessor
postproc = utils.find_entrypoint('turnstile.postprocessor',
proc,
compat=False)
if postproc:
# Note the reversed order
self.postprocessors.insert(0, postproc)
else:
# Using the classic syntax; grab preprocessors...
for preproc in self.conf.get('preprocess', '').split():
klass = utils.find_entrypoint('turnstile.preprocessor',
preproc,
required=True)
self.preprocessors.append(klass)
# And now the postprocessors...
for postproc in self.conf.get('postprocess', '').split():
klass = utils.find_entrypoint('turnstile.postprocessor',
postproc,
required=True)
self.postprocessors.append(klass)
# Set up the alternative formatter
formatter = self.conf.get('formatter')
if formatter:
formatter = utils.find_entrypoint('turnstile.formatter',
formatter,
required=True)
self.formatter = lambda a, b, c, d, e: formatter(
self.conf.status, a, b, c, d, e)
else:
self.formatter = self.format_delay
# Initialize the control daemon
if self.conf.to_bool(self.conf['control'].get('remote', 'no'), False):
self.control_daemon = remote.RemoteControlDaemon(self, self.conf)
else:
self.control_daemon = control.ControlDaemon(self, self.conf)
# Now start the control daemon
self.control_daemon.start()
# Emit a log message to indicate that we're running
LOG.info("Turnstile middleware initialized")