class TestConcurrentWindowManager(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(PersistenceHelper())
redis = yield self.persistence_helper.get_redis_manager()
self.window_id = 'window_id'
# Patch the count_waiting so we can fake the race condition
self.clock = Clock()
self.patch(WindowManager, 'count_waiting', lambda _, window_id: 100)
self.wm = WindowManager(redis, window_size=10, flight_lifetime=10)
self.add_cleanup(self.wm.stop)
yield self.wm.create_window(self.window_id)
self.redis = self.wm.redis
@inlineCallbacks
def test_race_condition(self):
"""
A race condition can occur when multiple window managers try and
access the same window at the same time.
A LoopingCall loops over the available windows, for those windows
it tries to get a next key. It does that by checking how many are
waiting to be sent out and adding however many it can still carry
to its own flight.
Since there are concurrent workers, between the time of checking how
many are available and how much room it has available, a different
window manager may have already beaten it to it.
If this happens Redis' `rpoplpush` method will return None since
there are no more available keys for the given window.
"""
yield self.wm.add(self.window_id, 1)
yield self.wm.add(self.window_id, 2)
yield self.wm._monitor_windows(lambda *a: True, True)
self.assertEqual((yield self.wm.get_next_key(self.window_id)), None)
class TestWindowManager(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(PersistenceHelper())
redis = yield self.persistence_helper.get_redis_manager()
self.window_id = 'window_id'
# Patch the clock so we can control time
self.clock = Clock()
self.patch(WindowManager, 'get_clock', lambda _: self.clock)
self.wm = WindowManager(redis, window_size=10, flight_lifetime=10)
self.add_cleanup(self.wm.stop)
yield self.wm.create_window(self.window_id)
self.redis = self.wm.redis
@inlineCallbacks
def test_windows(self):
windows = yield self.wm.get_windows()
self.assertTrue(self.window_id in windows)
def test_strict_window_recreation(self):
return self.assertFailure(
self.wm.create_window(self.window_id, strict=True),
WindowException)
@inlineCallbacks
def test_window_recreation(self):
orig_clock_time = self.clock.seconds()
clock_time = yield self.wm.create_window(self.window_id)
self.assertEqual(clock_time, orig_clock_time)
@inlineCallbacks
def test_window_removal(self):
yield self.wm.add(self.window_id, 1)
yield self.assertFailure(self.wm.remove_window(self.window_id),
WindowException)
key = yield self.wm.get_next_key(self.window_id)
item = yield self.wm.get_data(self.window_id, key)
self.assertEqual(item, 1)
self.assertEqual((yield self.wm.remove_window(self.window_id)), None)
@inlineCallbacks
def test_adding_to_window(self):
for i in range(10):
yield self.wm.add(self.window_id, i)
window_key = self.wm.window_key(self.window_id)
window_members = yield self.redis.llen(window_key)
self.assertEqual(window_members, 10)
@inlineCallbacks
def test_fetching_from_window(self):
for i in range(12):
yield self.wm.add(self.window_id, i)
flight_keys = []
for i in range(10):
flight_key = yield self.wm.get_next_key(self.window_id)
self.assertTrue(flight_key)
flight_keys.append(flight_key)
out_of_window_flight = yield self.wm.get_next_key(self.window_id)
self.assertEqual(out_of_window_flight, None)
# We should get data out in the order we put it in
for i, flight_key in enumerate(flight_keys):
data = yield self.wm.get_data(self.window_id, flight_key)
self.assertEqual(data, i)
# Removing one should allow for space for the next to fill up
yield self.wm.remove_key(self.window_id, flight_keys[0])
next_flight_key = yield self.wm.get_next_key(self.window_id)
self.assertTrue(next_flight_key)
@inlineCallbacks
def test_set_and_external_id(self):
yield self.wm.set_external_id(self.window_id, "flight_key",
"external_id")
self.assertEqual(
(yield self.wm.get_external_id(self.window_id, "flight_key")),
"external_id")
self.assertEqual(
(yield self.wm.get_internal_id(self.window_id, "external_id")),
"flight_key")
@inlineCallbacks
def test_remove_key_removes_external_and_internal_id(self):
yield self.wm.set_external_id(self.window_id, "flight_key",
"external_id")
yield self.wm.remove_key(self.window_id, "flight_key")
self.assertEqual(
(yield self.wm.get_external_id(self.window_id, "flight_key")),
None)
self.assertEqual(
(yield self.wm.get_internal_id(self.window_id, "external_id")),
None)
@inlineCallbacks
def assert_count_waiting(self, window_id, amount):
self.assertEqual((yield self.wm.count_waiting(window_id)), amount)
@inlineCallbacks
def assert_expired_keys(self, window_id, amount):
# Stuff has taken too long and so we should get 10 expired keys
expired_keys = yield self.wm.get_expired_flight_keys(window_id)
self.assertEqual(len(expired_keys), amount)
@inlineCallbacks
def assert_in_flight(self, window_id, amount):
self.assertEqual((yield self.wm.count_in_flight(window_id)),
amount)
@inlineCallbacks
def slide_window(self, limit=10):
for i in range(limit):
yield self.wm.get_next_key(self.window_id)
@inlineCallbacks
def test_expiry_of_acks(self):
def mock_clock_time(self):
return self._clocktime
self.patch(WindowManager, 'get_clocktime', mock_clock_time)
self.wm._clocktime = 0
for i in range(30):
yield self.wm.add(self.window_id, i)
# We're manually setting the clock instead of using clock.advance()
# so we can wait for the deferreds to finish before continuing to the
# next clear_expired_flight_keys run since LoopingCall() will only fire
# again if the previous run has completed.
yield self.slide_window()
self.wm._clocktime = 10
yield self.wm.clear_expired_flight_keys()
self.assert_expired_keys(self.window_id, 10)
yield self.slide_window()
self.wm._clocktime = 20
yield self.wm.clear_expired_flight_keys()
self.assert_expired_keys(self.window_id, 20)
yield self.slide_window()
self.wm._clocktime = 30
yield self.wm.clear_expired_flight_keys()
self.assert_expired_keys(self.window_id, 30)
self.assert_in_flight(self.window_id, 0)
self.assert_count_waiting(self.window_id, 0)
@inlineCallbacks
def test_monitor_windows(self):
yield self.wm.remove_window(self.window_id)
window_ids = ['window_id_1', 'window_id_2']
for window_id in window_ids:
yield self.wm.create_window(window_id)
for i in range(20):
yield self.wm.add(window_id, i)
key_callbacks = {}
def callback(window_id, key):
key_callbacks.setdefault(window_id, []).append(key)
cleanup_callbacks = []
def cleanup_callback(window_id):
cleanup_callbacks.append(window_id)
yield self.wm._monitor_windows(callback, False)
self.assertEqual(set(key_callbacks.keys()), set(window_ids))
self.assertEqual(len(key_callbacks.values()[0]), 10)
self.assertEqual(len(key_callbacks.values()[1]), 10)
yield self.wm._monitor_windows(callback, False)
# Nothing should've changed since we haven't removed anything.
self.assertEqual(len(key_callbacks.values()[0]), 10)
self.assertEqual(len(key_callbacks.values()[1]), 10)
for window_id, keys in key_callbacks.items():
for key in keys:
yield self.wm.remove_key(window_id, key)
yield self.wm._monitor_windows(callback, False)
# Everything should've been processed now
self.assertEqual(len(key_callbacks.values()[0]), 20)
self.assertEqual(len(key_callbacks.values()[1]), 20)
# Now run again but cleanup the empty windows
self.assertEqual(set((yield self.wm.get_windows())), set(window_ids))
for window_id, keys in key_callbacks.items():
for key in keys:
yield self.wm.remove_key(window_id, key)
yield self.wm._monitor_windows(callback, True, cleanup_callback)
self.assertEqual(len(key_callbacks.values()[0]), 20)
self.assertEqual(len(key_callbacks.values()[1]), 20)
self.assertEqual((yield self.wm.get_windows()), [])
self.assertEqual(set(cleanup_callbacks), set(window_ids))
