def test_discard(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.discard('foo')
self.assertNotIn('foo', s)
self.assertEqual(len(s._data), 0)
s.discard('foo')
def test_clear(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
self.assertEqual(len(s), 2)
s.clear()
self.assertFalse(s)
def test_purge(self):
# purge now enforces rules
# cant purge(1) now. but .purge(now=...) still works
s = LimitedSet(maxlen=10)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge()
self.assertEqual(len(s), 2)
# expired
s = LimitedSet(maxlen=10, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=time() + 100)
self.assertEqual(len(s), 0)
# not expired
s = LimitedSet(maxlen=None, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=lambda: time() - 100)
self.assertEqual(len(s), 2)
# expired -> minsize
s = LimitedSet(maxlen=10, minlen=10, expires=1)
[s.add(i) for i in range(20)]
s.minlen = 3
s.purge(now=time() + 3)
self.assertEqual(s.minlen, len(s))
self.assertLessEqual(
len(s._heap),
s.maxlen * (100. + s.max_heap_percent_overload) / 100,
)
def test_purge(self):
# purge now enforces rules
# cant purge(1) now. but .purge(now=...) still works
s = LimitedSet(maxlen=10)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge()
self.assertEqual(len(s), 2)
# expired
s = LimitedSet(maxlen=10, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=time() + 100)
self.assertEqual(len(s), 0)
# not expired
s = LimitedSet(maxlen=None, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=lambda: time() - 100)
self.assertEqual(len(s), 2)
# expired -> minsize
s = LimitedSet(maxlen=10, minlen=10, expires=1)
[s.add(i) for i in range(20)]
s.minlen = 3
s.purge(now=time() + 3)
self.assertEqual(s.minlen, len(s))
self.assertLessEqual(
len(s._heap),
s.maxlen * (100. + s.max_heap_percent_overload) / 100,
)
def test_clear(self):
s = LimitedSet(maxlen=2)
s.add("foo")
s.add("bar")
self.assertEqual(len(s), 2)
s.clear()
self.assertFalse(s)
def test_discard(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.discard('foo')
self.assertNotIn('foo', s)
self.assertEqual(len(s._data), 0)
s.discard('foo')
def test_start(self):
try:
c = Mock()
c.app.connection_for_read = _amqp_connection()
mingle = Mingle(c)
self.assertTrue(mingle.enabled)
Aig = LimitedSet()
Big = LimitedSet()
Aig.add("Aig-1")
Aig.add("Aig-2")
Big.add("Big-1")
I = c.app.control.inspect.return_value = Mock()
I.hello.return_value = {
"*****@*****.**": {"clock": 312, "revoked": Aig._data},
"*****@*****.**": {"clock": 29, "revoked": Big._data},
"*****@*****.**": {"error": "unknown method"},
}
mingle.start(c)
I.hello.assert_called_with(c.hostname, worker_state.revoked._data)
c.app.clock.adjust.assert_has_calls([call(312), call(29)], any_order=True)
self.assertIn("Aig-1", worker_state.revoked)
self.assertIn("Aig-2", worker_state.revoked)
self.assertIn("Big-1", worker_state.revoked)
finally:
worker_state.revoked.clear()
def test_add_removes_duplicate_from_small_heap(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('foo')
s.add('foo')
self.assertEqual(len(s), 1)
self.assertEqual(len(s._data), 1)
self.assertEqual(len(s._heap), 1)
def test_iter(self):
s = LimitedSet(maxlen=2)
items = "foo", "bar"
for item in items:
s.add(item)
l = list(iter(s))
for item in items:
self.assertIn(item, l)
def test_add_removes_duplicate_from_small_heap(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('foo')
s.add('foo')
self.assertEqual(len(s), 1)
self.assertEqual(len(s._data), 1)
self.assertEqual(len(s._heap), 1)
def test_iter(self):
s = LimitedSet(maxlen=2)
items = 'foo', 'bar'
for item in items:
s.add(item)
l = list(iter(s))
for item in items:
self.assertIn(item, l)
def test_add_removes_duplicate_from_big_heap(self):
s = LimitedSet(maxlen=1000)
[s.add(i) for i in range(2000)]
self.assertEqual(len(s), 1000)
[s.add('foo') for i in range(1000)]
# heap is refreshed when 15% larger than _data
self.assertLess(len(s._heap), 1150)
[s.add('foo') for i in range(1000)]
self.assertLess(len(s._heap), 1150)
def test_add_removes_duplicate_from_big_heap(self):
s = LimitedSet(maxlen=1000)
[s.add(i) for i in range(2000)]
self.assertEqual(len(s), 1000)
[s.add('foo') for i in range(1000)]
# heap is refreshed when 15% larger than _data
self.assertLess(len(s._heap), 1150)
[s.add('foo') for i in range(1000)]
self.assertLess(len(s._heap), 1150)
def test_iter(self):
s = LimitedSet(maxlen=3)
items = ['foo', 'bar', 'baz', 'xaz']
for item in items:
s.add(item)
l = list(iter(s))
for item in items[1:]:
self.assertIn(item, l)
self.assertNotIn('foo', l)
self.assertListEqual(l, items[1:], 'order by insertion time')
def test_pop_and_ordering_again(self):
s = LimitedSet(maxlen=5)
for i in range(10):
s.add(i)
j = -1
for _ in range(5):
i = s.pop()
self.assertLess(j, i)
i = s.pop()
self.assertEqual(i, None)
def test_add(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
self.assertIn(n, s)
s.add('baz')
for n in 'bar', 'baz':
self.assertIn(n, s)
self.assertNotIn('foo', s)
def test_add(self):
s = LimitedSet(maxlen=2)
s.add("foo")
s.add("bar")
for n in "foo", "bar":
self.assertTrue(n in s)
s.add("baz")
for n in "bar", "baz":
self.assertTrue(n in s)
self.assertTrue("foo" not in s)
def test_add(self):
s = LimitedSet(maxlen=2)
s.add("foo")
s.add("bar")
for n in "foo", "bar":
self.assertIn(n, s)
s.add("baz")
for n in "bar", "baz":
self.assertIn(n, s)
self.assertNotIn("foo", s)
def test_iter(self):
s = LimitedSet(maxlen=3)
items = ['foo', 'bar', 'baz', 'xaz']
for item in items:
s.add(item)
l = list(iter(s))
for item in items[1:]:
self.assertIn(item, l)
self.assertNotIn('foo', l)
self.assertListEqual(l, items[1:], 'order by insertion time')
def test_add(self):
s = LimitedSet(maxlen=2)
s.add("foo")
s.add("bar")
for n in "foo", "bar":
self.assertIn(n, s)
s.add("baz")
for n in "bar", "baz":
self.assertIn(n, s)
self.assertNotIn("foo", s)
def test_add(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
self.assertIn(n, s)
s.add('baz')
for n in 'bar', 'baz':
self.assertIn(n, s)
self.assertNotIn('foo', s)
def test_iter(self):
s = LimitedSet(maxlen=3)
items = ["foo", "bar", "baz", "xaz"]
for item in items:
s.add(item)
l = list(iter(s))
for item in items[1:]:
self.assertIn(item, l)
self.assertNotIn("foo", l)
self.assertListEqual(l, items[1:], "order by insertion time")
def test_pop_and_ordering_again(self):
s = LimitedSet(maxlen=5)
for i in range(10):
s.add(i)
j = -1
for _ in range(5):
i = s.pop()
self.assertLess(j, i)
i = s.pop()
self.assertEqual(i, None)
def test_add(self):
if sys.platform == 'win32':
raise SkipTest('Not working properly on Windows')
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
self.assertIn(n, s)
s.add('baz')
for n in 'bar', 'baz':
self.assertIn(n, s)
self.assertNotIn('foo', s)
def test_iterable_and_ordering(self):
s = LimitedSet(maxlen=35, expires=None)
# we use a custom clock here, as time.time() does not have enough
# precision when called quickly (can return the same value twice).
clock = count(1)
for i in reversed(range(15)):
s.add(i, now=next(clock))
j = 40
for i in s:
self.assertLess(i, j) # each item is smaller and smaller
j = i
self.assertEqual(i, 0) # last item is zero
def test_iterable_and_ordering(self):
s = LimitedSet(maxlen=35, expires=None)
# we use a custom clock here, as time.time() does not have enough
# precision when called quickly (can return the same value twice).
clock = count(1)
for i in reversed(range(15)):
s.add(i, now=next(clock))
j = 40
for i in s:
self.assertLess(i, j) # each item is smaller and smaller
j = i
self.assertEqual(i, 0) # last item is zero
def test_iter(self):
if sys.platform == 'win32':
raise SkipTest('Not working on Windows')
s = LimitedSet(maxlen=3)
items = ['foo', 'bar', 'baz', 'xaz']
for item in items:
s.add(item)
l = list(iter(s))
for item in items[1:]:
self.assertIn(item, l)
self.assertNotIn('foo', l)
self.assertListEqual(l, items[1:], 'order by insertion time')
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add('foo')
s1.add('bar')
s2 = LimitedSet(maxlen=2)
s2.update(s1)
self.assertItemsEqual(list(s2), ['foo', 'bar'])
s2.update(['bla'])
self.assertItemsEqual(list(s2), ['bla', 'bar'])
s2.update(['do', 're'])
self.assertItemsEqual(list(s2), ['do', 're'])
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add("foo")
s1.add("bar")
s2 = LimitedSet(maxlen=2)
s2.update(s1)
self.assertItemsEqual(list(s2), ["foo", "bar"])
s2.update(["bla"])
self.assertItemsEqual(list(s2), ["bla", "bar"])
s2.update(["do", "re"])
self.assertItemsEqual(list(s2), ["do", "re"])
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add('foo')
s1.add('bar')
s2 = LimitedSet(maxlen=2)
s2.update(s1)
self.assertItemsEqual(list(s2), ['foo', 'bar'])
s2.update(['bla'])
self.assertItemsEqual(list(s2), ['bla', 'bar'])
s2.update(['do', 're'])
self.assertItemsEqual(list(s2), ['do', 're'])
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add("foo")
s1.add("bar")
s2 = LimitedSet(maxlen=2)
s2.update(s1)
self.assertItemsEqual(list(s2), ["foo", "bar"])
s2.update(["bla"])
self.assertItemsEqual(list(s2), ["bla", "bar"])
s2.update(["do", "re"])
self.assertItemsEqual(list(s2), ["do", "re"])
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add('foo')
s1.add('bar')
s2 = LimitedSet(maxlen=2)
s2.update(s1)
self.assertItemsEqual(list(s2), ['foo', 'bar'])
s2.update(['bla'])
self.assertItemsEqual(list(s2), ['bla', 'bar'])
s2.update(['do', 're'])
self.assertItemsEqual(list(s2), ['do', 're'])
s1 = LimitedSet(maxlen=10, expires=None)
s2 = LimitedSet(maxlen=10, expires=None)
s3 = LimitedSet(maxlen=10, expires=None)
s4 = LimitedSet(maxlen=10, expires=None)
s5 = LimitedSet(maxlen=10, expires=None)
for i in range(12):
s1.add(i)
s2.add(i*i)
s3.update(s1)
s3.update(s2)
s4.update(s1.as_dict())
s4.update(s2.as_dict())
s5.update(s1._data) # revoke is using this
s5.update(s2._data)
self.assertEqual(s3, s4)
self.assertEqual(s3, s5)
s2.update(s4)
s4.update(s2)
self.assertEqual(s2, s4)
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add('foo')
s1.add('bar')
s2 = LimitedSet(maxlen=2)
s2.update(s1)
self.assertItemsEqual(list(s2), ['foo', 'bar'])
s2.update(['bla'])
self.assertItemsEqual(list(s2), ['bla', 'bar'])
s2.update(['do', 're'])
self.assertItemsEqual(list(s2), ['do', 're'])
s1 = LimitedSet(maxlen=10, expires=None)
s2 = LimitedSet(maxlen=10, expires=None)
s3 = LimitedSet(maxlen=10, expires=None)
s4 = LimitedSet(maxlen=10, expires=None)
s5 = LimitedSet(maxlen=10, expires=None)
for i in range(12):
s1.add(i)
s2.add(i * i)
s3.update(s1)
s3.update(s2)
s4.update(s1.as_dict())
s4.update(s2.as_dict())
s5.update(s1._data) # revoke is using this
s5.update(s2._data)
self.assertEqual(s3, s4)
self.assertEqual(s3, s5)
s2.update(s4)
s4.update(s2)
self.assertEqual(s2, s4)
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add("foo")
s1.add("bar")
s2 = LimitedSet(maxlen=2)
s2.update(s1)
self.assertItemsEqual(list(s2), ["foo", "bar"])
s2.update(["bla"])
self.assertItemsEqual(list(s2), ["bla", "bar"])
s2.update(["do", "re"])
self.assertItemsEqual(list(s2), ["do", "re"])
s1 = LimitedSet(maxlen=10, expires=None)
s2 = LimitedSet(maxlen=10, expires=None)
s3 = LimitedSet(maxlen=10, expires=None)
s4 = LimitedSet(maxlen=10, expires=None)
s5 = LimitedSet(maxlen=10, expires=None)
for i in range(12):
s1.add(i)
s2.add(i * i)
s3.update(s1)
s3.update(s2)
s4.update(s1.as_dict())
s4.update(s2.as_dict())
s5.update(s1._data) # revoke is using this
s5.update(s2._data)
self.assertEqual(s3, s4)
self.assertEqual(s3, s5)
s2.update(s4)
s4.update(s2)
self.assertEqual(s2, s4)
def test_start(self):
try:
c = Mock()
c.app.connection_for_read = _amqp_connection()
mingle = Mingle(c)
self.assertTrue(mingle.enabled)
Aig = LimitedSet()
Big = LimitedSet()
Aig.add('Aig-1')
Aig.add('Aig-2')
Big.add('Big-1')
I = c.app.control.inspect.return_value = Mock()
I.hello.return_value = {
'*****@*****.**': {
'clock': 312,
'revoked': Aig._data,
},
'*****@*****.**': {
'clock': 29,
'revoked': Big._data,
},
'*****@*****.**': {
'error': 'unknown method',
},
}
mingle.start(c)
I.hello.assert_called_with(c.hostname, worker_state.revoked._data)
c.app.clock.adjust.assert_has_calls([
call(312),
call(29),
],
any_order=True)
self.assertIn('Aig-1', worker_state.revoked)
self.assertIn('Aig-2', worker_state.revoked)
self.assertIn('Big-1', worker_state.revoked)
finally:
worker_state.revoked.clear()
def test_purge(self):
s = LimitedSet(maxlen=None)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(1)
self.assertEqual(len(s), 9)
s.purge(None)
self.assertEqual(len(s), 2)
# expired
s = LimitedSet(maxlen=None, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(1, now=lambda: time() + 100)
self.assertEqual(len(s), 9)
s.purge(None, now=lambda: time() + 100)
self.assertEqual(len(s), 2)
# not expired
s = LimitedSet(maxlen=None, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(1, now=lambda: time() - 100)
self.assertEqual(len(s), 10)
s.purge(None, now=lambda: time() - 100)
self.assertEqual(len(s), 10)
s = LimitedSet(maxlen=None)
[s.add(i) for i in range(10)]
s.maxlen = 2
with patch('celery.datastructures.heappop') as hp:
hp.side_effect = IndexError()
s.purge()
hp.assert_called_with(s._heap)
with patch('celery.datastructures.heappop') as hp:
s._data = {i * 2: i * 2 for i in range(10)}
s.purge()
self.assertEqual(hp.call_count, 10)
def test_start(self):
try:
c = Mock()
c.app.connection = _amqp_connection()
mingle = Mingle(c)
self.assertTrue(mingle.enabled)
Aig = LimitedSet()
Big = LimitedSet()
Aig.add('Aig-1')
Aig.add('Aig-2')
Big.add('Big-1')
I = c.app.control.inspect.return_value = Mock()
I.hello.return_value = {
'*****@*****.**': {
'clock': 312,
'revoked': Aig._data,
},
'*****@*****.**': {
'clock': 29,
'revoked': Big._data,
},
'*****@*****.**': {
'error': 'unknown method',
},
}
mingle.start(c)
I.hello.assert_called_with(c.hostname, worker_state.revoked._data)
c.app.clock.adjust.assert_has_calls([
call(312), call(29),
], any_order=True)
self.assertIn('Aig-1', worker_state.revoked)
self.assertIn('Aig-2', worker_state.revoked)
self.assertIn('Big-1', worker_state.revoked)
finally:
worker_state.revoked.clear()
def test_add(self):
s = LimitedSet(maxlen=2)
s.add("foo")
s.add("bar")
for n in "foo", "bar":
self.assertIn(n, s)
s.add("baz")
for n in "bar", "baz":
self.assertIn(n, s)
self.assertNotIn("foo", s)
s = LimitedSet(maxlen=10)
for i in range(150):
s.add(i)
self.assertLessEqual(len(s), 10)
# make sure heap is not leaking:
self.assertLessEqual(len(s._heap), len(s) * (100.0 + s.max_heap_percent_overload) / 100)
def test_add(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
self.assertIn(n, s)
s.add('baz')
for n in 'bar', 'baz':
self.assertIn(n, s)
self.assertNotIn('foo', s)
s = LimitedSet(maxlen=10)
for i in range(150):
s.add(i)
self.assertLessEqual(len(s), 10)
# make sure heap is not leaking:
self.assertLessEqual(
len(s._heap),
len(s) * (100. + s.max_heap_percent_overload) / 100,
)
def test_add(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
self.assertIn(n, s)
s.add('baz')
for n in 'bar', 'baz':
self.assertIn(n, s)
self.assertNotIn('foo', s)
s = LimitedSet(maxlen=10)
for i in range(150):
s.add(i)
self.assertLessEqual(len(s), 10)
# make sure heap is not leaking:
self.assertLessEqual(
len(s._heap),
len(s) * (100. + s.max_heap_percent_overload) / 100,
)
def test_add(self):
if sys.platform == 'win32':
raise SkipTest('Not working properly on Windows')
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
self.assertIn(n, s)
s.add('baz')
for n in 'bar', 'baz':
self.assertIn(n, s)
self.assertNotIn('foo', s)
s = LimitedSet(maxlen=10)
for i in range(150):
s.add(i)
self.assertLessEqual(len(s), 10)
# make sure heap is not leaking:
self.assertLessEqual(
len(s._heap),
len(s) * (100. + s.max_heap_percent_overload) / 100,
)
def test_add(self):
if sys.platform == 'win32':
raise SkipTest('Not working properly on Windows')
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
self.assertIn(n, s)
s.add('baz')
for n in 'bar', 'baz':
self.assertIn(n, s)
self.assertNotIn('foo', s)
s = LimitedSet(maxlen=10)
for i in range(150):
s.add(i)
self.assertLessEqual(len(s), 10)
# make sure heap is not leaking:
self.assertLessEqual(
len(s._heap),
len(s) * (100. + s.max_heap_percent_overload) / 100,
)
def test_repr(self):
s = LimitedSet(maxlen=2)
items = 'foo', 'bar'
for item in items:
s.add(item)
self.assertIn('LimitedSet(', repr(s))
def test_as_dict(self):
s = LimitedSet(maxlen=2)
s.add("foo")
self.assertIsInstance(s.as_dict(), dict)
def test_repr(self):
s = LimitedSet(maxlen=2)
items = "foo", "bar"
for item in items:
s.add(item)
self.assertIn("LimitedSet(", repr(s))
def test_as_dict(self):
s = LimitedSet(maxlen=2)
s.add('foo')
self.assertIsInstance(s.as_dict(), Mapping)
def test_pickleable(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
self.assertEqual(pickle.loads(pickle.dumps(s)), s)
def test_repr(self):
s = LimitedSet(maxlen=2)
items = 'foo', 'bar'
for item in items:
s.add(item)
self.assertIn('LimitedSet(', repr(s))
def test_discard(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.discard('foo')
self.assertNotIn('foo', s)
s.discard('foo')
def test_as_dict(self):
s = LimitedSet(maxlen=2)
s.add('foo')
self.assertIsInstance(s.as_dict(), dict)
def test_repr(self):
s = LimitedSet(maxlen=2)
items = "foo", "bar"
for item in items:
s.add(item)
self.assertIn("LimitedSet(", repr(s))
