def setUp(self):
super(TestEmptyQueue, self).setUp()
self.queue = FairQueue(priority_cb=self.priority_cb)
self.assertFalse(self.queue, "Queue did not initially evaluate False")
self.assertEqual(len(self.queue), 0, "Queue began with nonzero length")
self.assertFalse(self.queue._buckets, "Queue did not start empty")
self.assertFalse(self.queue._subqueues, "Queue did not start empty")
def setUp(self):
super(TestPopulatedQueue, self).setUp()
self.queue = FairQueue(priority_cb=self.priority_cb,
contents=self.users.values())
self._check_invariants()
self.assertTrue(self.queue, "Queue did not start out evaluating True")
self.assertTrue(len(self.queue), "Queue started out with zero length")
self.assertTrue(self.queue._buckets, "Queue started empty")
self.assertTrue(self.queue._subqueues, "Queue started empty")
self.assertTrue(all(x for x in self.queue._subqueues.values()),
"Queue started with empty subqueues")
def test_initial_data(self):
"""Test setting up a queue with initial data"""
data = self.users.values()
for dtype in (lambda x: x, list, dict, OrderedDict):
self.queue = FairQueue(
contents=dtype(data),
priority_cb=self.priority_cb)
self.assertTrue(all(x for x in self.queue._subqueues.values()),
"Queue was initialized with empty subqueues")
self._check_invariants(len(self.users))
self._check_equivalence(self.users)
# Check merging behaviour
self.queue = FairQueue(
contents=[(1, [1, 2]), (3, [3, 4]), (1, [3, 6])],
priority_cb=self.priority_cb)
self._check_invariants(2)
self._check_equivalence(FairQueue(
contents=[(1, [1, 2, 3, 6]), (3, [3, 4])]))
class TestPopulatedQueue(BaseTestQueue):
"""Tests which start with a populated queue"""
def setUp(self):
super(TestPopulatedQueue, self).setUp()
self.queue = FairQueue(priority_cb=self.priority_cb,
contents=self.users.values())
self._check_invariants()
self.assertTrue(self.queue, "Queue did not start out evaluating True")
self.assertTrue(len(self.queue), "Queue started out with zero length")
self.assertTrue(self.queue._buckets, "Queue started empty")
self.assertTrue(self.queue._subqueues, "Queue started empty")
self.assertTrue(all(x for x in self.queue._subqueues.values()),
"Queue started with empty subqueues")
def test_clear(self):
"""Test `FairQueue.clear`"""
self.queue.clear()
self.assertEqual(len(self.queue), 0, "Nonzero length after clear()")
self.assertFalse([x for x in self.queue],
"Content remains in queue after clear()")
def test_contains(self):
"""Test `FairQueue.__contains__` via ``key in queue``"""
for key in self.nonexistant_keys:
self.assertNotIn(key, self.queue,
"__contains__ should only return values actually in the queue")
for key in self.queue:
self.assertIn(key, self.queue,
"All entries returned by __iter__ should be 'in' the queue")
for key in self.users:
self.assertIn(key, self.queue,
"All entries added to the queue should be 'in' it")
def test_dump_load_symmetry(self):
"""Test symmetry of `FairQueue.dump` and `FairQueue.load`"""
dump = self.queue.dump()
newQueue = self.queue.load(dump)
self._check_equivalence(newQueue)
newQueue = self.queue.load(dump, priority_cb=self.queue.priority_cb)
self.assertIs(self.queue.priority_cb, newQueue.priority_cb)
self._check_equivalence(newQueue)
self.assertIsNot(self.queue._buckets, newQueue._buckets,
"dump() must copy the bucket heap")
self.assertIsNot(self.queue._subqueues, newQueue._subqueues,
"dump() must copy the subqueue list")
def test_delitem(self):
"""Test `FairQueue.__delitem__`"""
for key in self.nonexistant_keys:
self.assertRaises(KeyError, self.queue.__delitem__, key)
for user in self.users:
self.assertIn(user, self.queue)
del self.queue[user]
self._check_invariants()
self.assertNotIn(user, self.queue,
"'del self.queue[user]' must remove 'user' from the queue")
def test_getitem(self):
"""Test `FairQueue.__getitem__`"""
# Just to make sure it doesn't behave differently than an empty queue
for key in self.nonexistant_keys:
self.assertRaises(KeyError, self.queue.__getitem__, key)
for user in self.users.values():
bucket = self.queue[user.bucket_id]
self.assertEqual(bucket, user.goal)
self._check_invariants()
bucket[0], bucket[1] = bucket[1], bucket[0]
self.assertEqual(bucket, self.queue[user.bucket_id],
"Mutations to the list returned by __getitem__ must persist.")
goal_item = bucket[0]
self.assertEqual(self.queue.pop(user.bucket_id)[1], goal_item,
"Mutations to lists returned by __getitem__ must affect pop()")
def test_getitem_safety(self):
"""Test emptying a subqueue using `FairQueue.__getitem__`"""
# Empty the next subqueue in line
key = self.queue._buckets[0][1]
bucket = self.queue[key]
while bucket:
bucket.pop()
self.assertFalse(self.queue[key])
# Force queue expiry
next_id = self.queue._buckets[1][1]
self.assertEqual(self.queue.pop()[0], next_id,
"Unexpected return from pop() when passing through empty subqueue")
# Verify the cleanup occurred
self._check_invariants()
def test_invalid_pop(self):
"""Test reaction to `FairQueue.pop` on populated queue"""
for key in self.nonexistant_keys:
self.assertRaises(KeyError, self.queue.pop, key)
def test_iter(self):
"""Test queue iteration"""
#TODO: Once we've got DictMixin, use assertEqual directly on queues.
self.assertEqual([x for x in self.queue], [x for x in self.users],
"Queue iteration must return a list of keys (subqueue names)")
self.queue.push('foo', [1, 2, 3])
self.queue.push('bar', [1, 2, 3])
self.assertEqual(list(self.queue), list(self.users) + ['foo', 'bar'],
"Queue iteration must be in priority order")
def test_keys(self):
"""Test `FairQueue.keys`"""
self.assertEqual(self.queue.keys(), [x for x in self.queue],
"keys() must return the same sequence as iteration")
def test_len(self):
"""Test `FairQueue.__len__` via ``len(queue)``"""
total_entries = sum(len(self.queue[x]) for x in self.queue)
self.assertEqual(len(self.queue), total_entries,
"Length of the queue must sum the lengths of its subqueues")
def test_nonzero(self):
"""Test all branches of `FairQueue.__nonzero__`"""
self.assertTrue(self.queue)
# Test the hardest-to-reach branch
for key in self.queue: # pragma: no branch
self.queue[key] = []
break
self.assertTrue(self.queue)
for key in self.queue:
self.queue[key] = []
self.assertFalse(self.queue)
self.queue.clear()
self.assertFalse(self.queue)
def test_ordering_basic(self):
"""Test ordering behaviour with only non-specific pop() calls"""
previous = None
for i in range(0, len(self.queue)):
key, value = self.queue.pop()
self.assertNotEqual(key, previous,
"At this stage in its development, the queue should never "
"allow the same bucket to be the source of two pop() calls"
" in a row when all subqueues are of equal length")
def test_ordering_advanced(self):
"""Test ordering behaviour with subqueue-specific pop() calls"""
ordered_heap = list(sorted(self.queue))
for pos in (0, -1, len(ordered_heap) // 2, None, Ellipsis):
if pos is None:
key, value = self.queue.pop()
elif pos is Ellipsis:
key, value = ('foo', ['bar', 'baz'])
else:
key, value = self.queue.pop(ordered_heap[pos])
self.queue.push(key, value)
last_user = list(self.queue)[-1]
self.assertEqual(key, last_user,
"Users who've just been serviced/added should be at the end of"
" queue (Last user was %r but expected %r)" % (last_user, key))
def test_pop(self):
"""Test behaviour of valid `FairQueue.pop` calls without ``key``"""
while self.queue:
before = len(self.queue)
old_hash_record = self.queue._buckets[0]
key, value = self.queue.pop()
self.assertNotIn(old_hash_record, self.queue._buckets,
"Pop must always update a bucket's hash ordering key")
self.assertEqual(before - 1, len(self.queue))
self._check_invariants()
self.users[key].received(value)
unsatisfied = [x for x in self.users.values() if not x.is_satisfied()]
self.assertFalse(unsatisfied,
'Not all users received their files:\n\t%s' %
'\n\t'.join('%s: %s, %s' % (x.bucket_id, x.goal, x.done)
for x in unsatisfied))
# Verify deferred subqueue removal
self._check_invariants()
def test_pop_with_id(self):
"""Test behaviour of `FairQueue.pop` calls with ``key`` parameter"""
for key in self.nonexistant_keys:
self.assertRaises(KeyError, self.queue.pop, key)
for user in self.users.values():
while user.bucket_id in self.queue:
for entry in self.queue._buckets: # pragma: no branch
if entry[1] == user.bucket_id:
old_hash_record = entry
break
key, value = self.queue.pop(key=user.bucket_id)
self.assertEqual(key, user.bucket_id)
self.assertNotIn(old_hash_record, self.queue._buckets,
"Pop must always update a bucket's hash ordering key")
user.received(value)
self.assertTrue(user.is_satisfied(),
"pop(id) failed to retrieve all push(id, ...)'d items")
# Verify deferred subqueue removal
self._check_invariants()
def test_pop_safety(self):
"""Test pop() when heap and subqueues are inconsistent"""
# Test the hardest-to-reach branch in pop()
for key in self.queue: # pragma: no branch
del self.queue._subqueues[key]
break
self.queue.pop()
def test_setitem(self):
"""Test `FairQueue.__setitem__`"""
test_key, test_value = 'foo', [1, 2, 3]
before_value = test_value[:]
after_value = test_value[1:]
self.queue['foo'] = test_value
self.assertEqual(self.queue['foo'], before_value,
"__setitem__ should insert new subqueues without modification")
self.assertEqual(self.queue.pop('foo'), ('foo', 1),
"pop('new_key') must behave as expected after __setitem__")
self.assertEqual(self.queue['foo'], after_value,
"pop('new_key') must affect future __getitem__ calls")
self.assertEqual(self.queue['foo'], test_value,
"If at all possible, existing references to __setitem__'s "
"input must remain as mutable references to the subqueue")
class TestEmptyQueue(BaseTestQueue):
"""Tests which start with an empty queue"""
def setUp(self):
super(TestEmptyQueue, self).setUp()
self.queue = FairQueue(priority_cb=self.priority_cb)
self.assertFalse(self.queue, "Queue did not initially evaluate False")
self.assertEqual(len(self.queue), 0, "Queue began with nonzero length")
self.assertFalse(self.queue._buckets, "Queue did not start empty")
self.assertFalse(self.queue._subqueues, "Queue did not start empty")
def test_initial_data(self):
"""Test setting up a queue with initial data"""
data = self.users.values()
for dtype in (lambda x: x, list, dict, OrderedDict):
self.queue = FairQueue(
contents=dtype(data),
priority_cb=self.priority_cb)
self.assertTrue(all(x for x in self.queue._subqueues.values()),
"Queue was initialized with empty subqueues")
self._check_invariants(len(self.users))
self._check_equivalence(self.users)
# Check merging behaviour
self.queue = FairQueue(
contents=[(1, [1, 2]), (3, [3, 4]), (1, [3, 6])],
priority_cb=self.priority_cb)
self._check_invariants(2)
self._check_equivalence(FairQueue(
contents=[(1, [1, 2, 3, 6]), (3, [3, 4])]))
def test_invalid_pop(self):
"""Test reaction to `FairQueue.pop` on empty queue"""
self.assertRaises(IndexError, self.queue.pop)
for key in self.nonexistant_keys:
self.assertRaises(KeyError, self.queue.pop, key)
def test_invalid_getitem(self):
"""Test reaction to `FairQueue.__getitem__` on a nonexistant ID"""
for key in self.nonexistant_keys:
self.assertRaises(KeyError, self.queue.__getitem__, key)
def test_invalid_load(self):
"""Test reaction to invalid `FairQueue.load` input"""
self.assertRaises(TypeError, FairQueue.load, ({}, {}))
self.assertRaises(TypeError, FairQueue.load, ([], []))
def test_invalid_push(self):
"""Test reaction to unhashable ``key`` in `FairQueue.push`"""
self.assertRaises(TypeError, self.queue.push, {}, None)
self.assertFalse(self.queue._buckets,
"Refusal of invalid keys must not alter the bucket heap")
self.assertFalse(self.queue._subqueues,
"Refusal of invalid keys must not alter the subqueue dict")
# Now make sure it doesn't bother existing content
self.queue.push(1, 2)
_b, _q = self.queue._buckets[:], self.queue._subqueues.copy()
self.assertRaises(TypeError, self.queue.push, {}, None)
self.assertEqual(_b, self.queue._buckets,
"Refusal of invalid keys must not alter the bucket heap")
self.assertEqual(_q, self.queue._subqueues,
"Refusal of invalid keys must not alter the subqueue dict")
def test_push(self):
"""Test behaviour of valid `FairQueue.push` calls"""
target_count = 0
for user in self.users.values():
self._check_invariants(target_count)
#TODO: Test with mock time.time() in case of low-resolution timers.
for entry in iter(user.request, None):
before = len(self.queue)
self.queue.push(user.bucket_id, entry)
self.assertEqual(before + 1, len(self.queue))
target_count += 1
self._check_equivalence(self.users)
def test_populate_equivalence(self):
"""Test that `FairQueue.__init__` and `FairQueue.push` order equally"""
populated_queue = FairQueue(
contents=self.users.values(),
priority_cb=self.priority_cb)
for user in self.users.values():
for entry in iter(user.request, None):
self.queue.push(user.bucket_id, entry)
self._check_equivalence(populated_queue)