def test_hello__with_revoked(self):
revoked = LimitedSet(100)
for i in range(100):
revoked.add(f'id{i}')
self.inspect.hello('*****@*****.**', revoked=revoked._data)
self.assert_broadcast_called(
'hello', from_node='*****@*****.**', revoked=revoked._data)
def test_clear(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
self.assertEqual(len(s), 2)
s.clear()
self.assertFalse(s)
def test_hello__with_revoked(self):
revoked = LimitedSet(100)
for i in range(100):
revoked.add('id{0}'.format(i))
self.inspect.hello('*****@*****.**', revoked=revoked._data)
self.assert_broadcast_called(
'hello', from_node='*****@*****.**', revoked=revoked._data)
def test_clear(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
assert len(s) == 2
s.clear()
assert not s
def test_add_removes_duplicate_from_small_heap(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('foo')
s.add('foo')
self.assertEqual(len(s), 1)
self.assertEqual(len(s._data), 1)
self.assertEqual(len(s._heap), 1)
def test_add_removes_duplicate_from_small_heap(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('foo')
s.add('foo')
assert len(s) == 1
assert len(s._data) == 1
assert len(s._heap) == 1
def test_add_removes_duplicate_from_big_heap(self):
s = LimitedSet(maxlen=1000)
[s.add(i) for i in range(2000)]
self.assertEqual(len(s), 1000)
[s.add('foo') for i in range(1000)]
# heap is refreshed when 15% larger than _data
self.assertLess(len(s._heap), 1150)
[s.add('foo') for i in range(1000)]
self.assertLess(len(s._heap), 1150)
def test_add_removes_duplicate_from_big_heap(self):
s = LimitedSet(maxlen=1000)
[s.add(i) for i in range(2000)]
assert len(s) == 1000
[s.add('foo') for i in range(1000)]
# heap is refreshed when 15% larger than _data
assert len(s._heap) < 1150
[s.add('foo') for i in range(1000)]
assert len(s._heap) < 1150
def test_add_removes_duplicate_from_big_heap(self):
s = LimitedSet(maxlen=1000)
[s.add(i) for i in range(2000)]
assert len(s) == 1000
[s.add('foo') for i in range(1000)]
# heap is refreshed when 15% larger than _data
assert len(s._heap) < 1150
[s.add('foo') for i in range(1000)]
assert len(s._heap) < 1150
def test_add_removes_duplicate_from_big_heap(self):
s = LimitedSet(maxlen=1000)
[s.add(i) for i in range(2000)]
self.assertEqual(len(s), 1000)
[s.add('foo') for i in range(1000)]
# heap is refreshed when 15% larger than _data
self.assertLess(len(s._heap), 1150)
[s.add('foo') for i in range(1000)]
self.assertLess(len(s._heap), 1150)
def test_iter(self):
s = LimitedSet(maxlen=3)
items = ['foo', 'bar', 'baz', 'xaz']
for item in items:
s.add(item)
l = list(iter(s))
for item in items[1:]:
self.assertIn(item, l)
self.assertNotIn('foo', l)
self.assertListEqual(l, items[1:], 'order by insertion time')
def test_iter(self):
s = LimitedSet(maxlen=3)
items = ['foo', 'bar', 'baz', 'xaz']
for item in items:
s.add(item)
l = list(iter(s))
for item in items[1:]:
assert item in l
assert 'foo' not in l
assert l == items[1:], 'order by insertion time'
def test_pop_and_ordering_again(self):
s = LimitedSet(maxlen=5)
for i in range(10):
s.add(i)
j = -1
for _ in range(5):
i = s.pop()
assert j < i
i = s.pop()
assert i is None
def test_iterable_and_ordering(self):
s = LimitedSet(maxlen=35, expires=None)
clock = count(1)
for i in reversed(range(15)):
s.add(i, now=next(clock))
j = 40
for i in s:
assert i < j # each item is smaller and smaller
j = i
assert i == 0 # last item is zero
def test_iterable_and_ordering(self):
s = LimitedSet(maxlen=35, expires=None)
clock = count(1)
for i in reversed(range(15)):
s.add(i, now=next(clock))
j = 40
for i in s:
assert i < j # each item is smaller and smaller
j = i
assert i == 0 # last item is zero
def test_iter(self):
s = LimitedSet(maxlen=3)
items = ['foo', 'bar', 'baz', 'xaz']
for item in items:
s.add(item)
l = list(iter(s))
for item in items[1:]:
assert item in l
assert 'foo' not in l
assert l == items[1:], 'order by insertion time'
def test_iter(self):
s = LimitedSet(maxlen=3)
items = ['foo', 'bar', 'baz', 'xaz']
for item in items:
s.add(item)
l = list(iter(s))
for item in items[1:]:
self.assertIn(item, l)
self.assertNotIn('foo', l)
self.assertListEqual(l, items[1:], 'order by insertion time')
def test_pop_and_ordering_again(self):
s = LimitedSet(maxlen=5)
for i in range(10):
s.add(i)
j = -1
for _ in range(5):
i = s.pop()
self.assertLess(j, i)
i = s.pop()
self.assertEqual(i, None)
def test_iterable_and_ordering(self):
s = LimitedSet(maxlen=35, expires=None)
# we use a custom clock here, as time.time() does not have enough
# precision when called quickly (can return the same value twice).
clock = count(1)
for i in reversed(range(15)):
s.add(i, now=next(clock))
j = 40
for i in s:
self.assertLess(i, j) # each item is smaller and smaller
j = i
self.assertEqual(i, 0) # last item is zero
def test_iterable_and_ordering(self):
s = LimitedSet(maxlen=35, expires=None)
# we use a custom clock here, as time.time() does not have enough
# precision when called quickly (can return the same value twice).
clock = count(1)
for i in reversed(range(15)):
s.add(i, now=next(clock))
j = 40
for i in s:
assert i < j # each item is smaller and smaller
j = i
assert i == 0 # last item is zero
def test_discard(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.discard('foo')
assert 'foo' not in s
assert len(s._data) == 0
s.discard('foo')
def test_discard(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.discard('foo')
self.assertNotIn('foo', s)
self.assertEqual(len(s._data), 0)
s.discard('foo')
def test_discard(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.discard('foo')
self.assertNotIn('foo', s)
self.assertEqual(len(s._data), 0)
s.discard('foo')
def test_discard(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.discard('foo')
assert 'foo' not in s
assert len(s._data) == 0
s.discard('foo')
def test_add(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
assert n in s
s.add('baz')
for n in 'bar', 'baz':
assert n in s
assert 'foo' not in s
s = LimitedSet(maxlen=10)
for i in range(150):
s.add(i)
assert len(s) <= 10
# make sure heap is not leaking:
assert len(
s._heap) < len(s) * (100. + s.max_heap_percent_overload) / 100
def test_add(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
self.assertIn(n, s)
s.add('baz')
for n in 'bar', 'baz':
self.assertIn(n, s)
self.assertNotIn('foo', s)
s = LimitedSet(maxlen=10)
for i in range(150):
s.add(i)
self.assertLessEqual(len(s), 10)
# make sure heap is not leaking:
self.assertLessEqual(
len(s._heap),
len(s) * (100. + s.max_heap_percent_overload) / 100,
)
def test_clear(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
assert len(s) == 2
s.clear()
assert not s
def test_clear(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
self.assertEqual(len(s), 2)
s.clear()
self.assertFalse(s)
def test_add_removes_duplicate_from_small_heap(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('foo')
s.add('foo')
self.assertEqual(len(s), 1)
self.assertEqual(len(s._data), 1)
self.assertEqual(len(s._heap), 1)
def test_add_removes_duplicate_from_small_heap(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('foo')
s.add('foo')
assert len(s) == 1
assert len(s._data) == 1
assert len(s._heap) == 1
def test_pop_and_ordering_again(self):
s = LimitedSet(maxlen=5)
for i in range(10):
s.add(i)
j = -1
for _ in range(5):
i = s.pop()
assert j < i
i = s.pop()
assert i is None
def test_pop_and_ordering_again(self):
s = LimitedSet(maxlen=5)
for i in range(10):
s.add(i)
j = -1
for _ in range(5):
i = s.pop()
self.assertLess(j, i)
i = s.pop()
self.assertEqual(i, None)
def test_start(self):
c = Mock()
c.app.connection_for_read = _amqp_connection()
mingle = Mingle(c)
assert mingle.enabled
Aig = LimitedSet()
Big = LimitedSet()
Aig.add('Aig-1')
Aig.add('Aig-2')
Big.add('Big-1')
I = c.app.control.inspect.return_value = Mock()
I.hello.return_value = {
'*****@*****.**': {
'clock': 312,
'revoked': Aig._data,
},
'*****@*****.**': {
'clock': 29,
'revoked': Big._data,
},
'*****@*****.**': {
'error': 'unknown method',
},
}
our_revoked = c.controller.state.revoked = LimitedSet()
mingle.start(c)
I.hello.assert_called_with(c.hostname, our_revoked._data)
c.app.clock.adjust.assert_has_calls([
call(312),
call(29),
],
any_order=True)
assert 'Aig-1' in our_revoked
assert 'Aig-2' in our_revoked
assert 'Big-1' in our_revoked
def test_add(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
assert n in s
s.add('baz')
for n in 'bar', 'baz':
assert n in s
assert 'foo' not in s
s = LimitedSet(maxlen=10)
for i in range(150):
s.add(i)
assert len(s) <= 10
# make sure heap is not leaking:
assert len(s._heap) < len(s) * (
100. + s.max_heap_percent_overload) / 100
def test_add(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
for n in 'foo', 'bar':
self.assertIn(n, s)
s.add('baz')
for n in 'bar', 'baz':
self.assertIn(n, s)
self.assertNotIn('foo', s)
s = LimitedSet(maxlen=10)
for i in range(150):
s.add(i)
self.assertLessEqual(len(s), 10)
# make sure heap is not leaking:
self.assertLessEqual(
len(s._heap),
len(s) * (100. + s.max_heap_percent_overload) / 100,
)
def test_start(self):
c = Mock()
c.app.connection_for_read = _amqp_connection()
mingle = Mingle(c)
self.assertTrue(mingle.enabled)
Aig = LimitedSet()
Big = LimitedSet()
Aig.add('Aig-1')
Aig.add('Aig-2')
Big.add('Big-1')
I = c.app.control.inspect.return_value = Mock()
I.hello.return_value = {
'*****@*****.**': {
'clock': 312,
'revoked': Aig._data,
},
'*****@*****.**': {
'clock': 29,
'revoked': Big._data,
},
'*****@*****.**': {
'error': 'unknown method',
},
}
our_revoked = c.controller.state.revoked = LimitedSet()
mingle.start(c)
I.hello.assert_called_with(c.hostname, our_revoked._data)
c.app.clock.adjust.assert_has_calls([
call(312), call(29),
], any_order=True)
self.assertIn('Aig-1', our_revoked)
self.assertIn('Aig-2', our_revoked)
self.assertIn('Big-1', our_revoked)
def test_repr(self):
s = LimitedSet(maxlen=2)
items = 'foo', 'bar'
for item in items:
s.add(item)
self.assertIn('LimitedSet(', repr(s))
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add('foo')
s1.add('bar')
s2 = LimitedSet(maxlen=2)
s2.update(s1)
self.assertItemsEqual(list(s2), ['foo', 'bar'])
s2.update(['bla'])
self.assertItemsEqual(list(s2), ['bla', 'bar'])
s2.update(['do', 're'])
self.assertItemsEqual(list(s2), ['do', 're'])
s1 = LimitedSet(maxlen=10, expires=None)
s2 = LimitedSet(maxlen=10, expires=None)
s3 = LimitedSet(maxlen=10, expires=None)
s4 = LimitedSet(maxlen=10, expires=None)
s5 = LimitedSet(maxlen=10, expires=None)
for i in range(12):
s1.add(i)
s2.add(i * i)
s3.update(s1)
s3.update(s2)
s4.update(s1.as_dict())
s4.update(s2.as_dict())
s5.update(s1._data) # revoke is using this
s5.update(s2._data)
self.assertEqual(s3, s4)
self.assertEqual(s3, s5)
s2.update(s4)
s4.update(s2)
self.assertEqual(s2, s4)
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add('foo')
s1.add('bar')
s2 = LimitedSet(maxlen=2)
s2.update(s1)
assert sorted(list(s2)) == ['bar', 'foo']
s2.update(['bla'])
assert sorted(list(s2)) == ['bar', 'bla']
s2.update(['do', 're'])
assert sorted(list(s2)) == ['do', 're']
s1 = LimitedSet(maxlen=10, expires=None)
s2 = LimitedSet(maxlen=10, expires=None)
s3 = LimitedSet(maxlen=10, expires=None)
s4 = LimitedSet(maxlen=10, expires=None)
s5 = LimitedSet(maxlen=10, expires=None)
for i in range(12):
s1.add(i)
s2.add(i * i)
s3.update(s1)
s3.update(s2)
s4.update(s1.as_dict())
s4.update(s2.as_dict())
s5.update(s1._data) # revoke is using this
s5.update(s2._data)
assert s3 == s4
assert s3 == s5
s2.update(s4)
s4.update(s2)
assert s2 == s4
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add('foo')
s1.add('bar')
s2 = LimitedSet(maxlen=2)
s2.update(s1)
self.assertItemsEqual(list(s2), ['foo', 'bar'])
s2.update(['bla'])
self.assertItemsEqual(list(s2), ['bla', 'bar'])
s2.update(['do', 're'])
self.assertItemsEqual(list(s2), ['do', 're'])
s1 = LimitedSet(maxlen=10, expires=None)
s2 = LimitedSet(maxlen=10, expires=None)
s3 = LimitedSet(maxlen=10, expires=None)
s4 = LimitedSet(maxlen=10, expires=None)
s5 = LimitedSet(maxlen=10, expires=None)
for i in range(12):
s1.add(i)
s2.add(i * i)
s3.update(s1)
s3.update(s2)
s4.update(s1.as_dict())
s4.update(s2.as_dict())
s5.update(s1._data) # revoke is using this
s5.update(s2._data)
self.assertEqual(s3, s4)
self.assertEqual(s3, s5)
s2.update(s4)
s4.update(s2)
self.assertEqual(s2, s4)
def test_purge(self):
# purge now enforces rules
# cant purge(1) now. but .purge(now=...) still works
s = LimitedSet(maxlen=10)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge()
assert len(s) == 2
# expired
s = LimitedSet(maxlen=10, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=time() + 100)
assert len(s) == 0
# not expired
s = LimitedSet(maxlen=None, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=lambda: time() - 100)
assert len(s) == 2
# expired -> minsize
s = LimitedSet(maxlen=10, minlen=10, expires=1)
[s.add(i) for i in range(20)]
s.minlen = 3
s.purge(now=time() + 3)
assert s.minlen == len(s)
assert len(s._heap) <= s.maxlen * (
100. + s.max_heap_percent_overload) / 100
def test_repr(self):
s = LimitedSet(maxlen=2)
items = 'foo', 'bar'
for item in items:
s.add(item)
assert 'LimitedSet(' in repr(s)
def test_repr(self):
s = LimitedSet(maxlen=2)
items = 'foo', 'bar'
for item in items:
s.add(item)
self.assertIn('LimitedSet(', repr(s))
def test_pickleable(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
assert pickle.loads(pickle.dumps(s)) == s
def test_purge(self):
# purge now enforces rules
# cant purge(1) now. but .purge(now=...) still works
s = LimitedSet(maxlen=10)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge()
assert len(s) == 2
# expired
s = LimitedSet(maxlen=10, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=monotonic() + 100)
assert len(s) == 0
# not expired
s = LimitedSet(maxlen=None, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=lambda: monotonic() - 100)
assert len(s) == 2
# expired -> minsize
s = LimitedSet(maxlen=10, minlen=10, expires=1)
[s.add(i) for i in range(20)]
s.minlen = 3
s.purge(now=monotonic() + 3)
assert s.minlen == len(s)
assert len(
s._heap) <= s.maxlen * (100. + s.max_heap_percent_overload) / 100
def test_repr(self):
s = LimitedSet(maxlen=2)
items = 'foo', 'bar'
for item in items:
s.add(item)
assert 'LimitedSet(' in repr(s)
def add_revoked(self, p, *ids):
for id in ids:
p.db.setdefault('revoked', LimitedSet()).add(id)
requests = {}
#: set of all reserved :class:`~celery.worker.request.Request`'s.
reserved_requests = weakref.WeakSet()
#: set of currently active :class:`~celery.worker.request.Request`'s.
active_requests = weakref.WeakSet()
#: count of tasks accepted by the worker, sorted by type.
total_count = Counter()
#: count of all tasks accepted by the worker
all_total_count = [0]
#: the list of currently revoked tasks. Persistent if ``statedb`` set.
revoked = LimitedSet(maxlen=REVOKES_MAX, expires=REVOKE_EXPIRES)
should_stop = None
should_terminate = None
def reset_state():
requests.clear()
reserved_requests.clear()
active_requests.clear()
total_count.clear()
all_total_count[:] = [0]
revoked.clear()
def maybe_shutdown():
def test_update(self):
s1 = LimitedSet(maxlen=2)
s1.add('foo')
s1.add('bar')
s2 = LimitedSet(maxlen=2)
s2.update(s1)
assert sorted(list(s2)) == ['bar', 'foo']
s2.update(['bla'])
assert sorted(list(s2)) == ['bar', 'bla']
s2.update(['do', 're'])
assert sorted(list(s2)) == ['do', 're']
s1 = LimitedSet(maxlen=10, expires=None)
s2 = LimitedSet(maxlen=10, expires=None)
s3 = LimitedSet(maxlen=10, expires=None)
s4 = LimitedSet(maxlen=10, expires=None)
s5 = LimitedSet(maxlen=10, expires=None)
for i in range(12):
s1.add(i)
s2.add(i * i)
s3.update(s1)
s3.update(s2)
s4.update(s1.as_dict())
s4.update(s2.as_dict())
s5.update(s1._data) # revoke is using this
s5.update(s2._data)
assert s3 == s4
assert s3 == s5
s2.update(s4)
s4.update(s2)
assert s2 == s4
def test_as_dict(self):
s = LimitedSet(maxlen=2)
s.add('foo')
assert isinstance(s.as_dict(), Mapping)
def test_pickleable(self):
s = LimitedSet(maxlen=2)
s.add('foo')
s.add('bar')
assert pickle.loads(pickle.dumps(s)) == s
def test_as_dict(self):
s = LimitedSet(maxlen=2)
s.add('foo')
assert isinstance(s.as_dict(), Mapping)
def test_purge(self):
# purge now enforces rules
# cant purge(1) now. but .purge(now=...) still works
s = LimitedSet(maxlen=10)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge()
self.assertEqual(len(s), 2)
# expired
s = LimitedSet(maxlen=10, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=time() + 100)
self.assertEqual(len(s), 0)
# not expired
s = LimitedSet(maxlen=None, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=lambda: time() - 100)
self.assertEqual(len(s), 2)
# expired -> minsize
s = LimitedSet(maxlen=10, minlen=10, expires=1)
[s.add(i) for i in range(20)]
s.minlen = 3
s.purge(now=time() + 3)
self.assertEqual(s.minlen, len(s))
self.assertLessEqual(
len(s._heap),
s.maxlen * (100. + s.max_heap_percent_overload) / 100,
)
#: how many seconds a successful task will be cached in memory
#: before being expired when the max limit has been exceeded.
SUCCESSFUL_EXPIRES = 10800
#: Mapping of reserved task_id->Request.
requests = {}
#: set of all reserved :class:`~celery.worker.request.Request`'s.
reserved_requests = weakref.WeakSet()
#: set of currently active :class:`~celery.worker.request.Request`'s.
active_requests = weakref.WeakSet()
#: A limited set of successful :class:`~celery.worker.request.Request`'s.
successful_requests = LimitedSet(maxlen=SUCCESSFUL_MAX,
expires=SUCCESSFUL_EXPIRES)
#: count of tasks accepted by the worker, sorted by type.
total_count = Counter()
#: count of all tasks accepted by the worker
all_total_count = [0]
#: the list of currently revoked tasks. Persistent if ``statedb`` set.
revoked = LimitedSet(maxlen=REVOKES_MAX, expires=REVOKE_EXPIRES)
should_stop = None
should_terminate = None
def reset_state():
def test_purge(self):
# purge now enforces rules
# cant purge(1) now. but .purge(now=...) still works
s = LimitedSet(maxlen=10)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge()
self.assertEqual(len(s), 2)
# expired
s = LimitedSet(maxlen=10, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=time() + 100)
self.assertEqual(len(s), 0)
# not expired
s = LimitedSet(maxlen=None, expires=1)
[s.add(i) for i in range(10)]
s.maxlen = 2
s.purge(now=lambda: time() - 100)
self.assertEqual(len(s), 2)
# expired -> minsize
s = LimitedSet(maxlen=10, minlen=10, expires=1)
[s.add(i) for i in range(20)]
s.minlen = 3
s.purge(now=time() + 3)
self.assertEqual(s.minlen, len(s))
self.assertLessEqual(
len(s._heap),
s.maxlen * (100. + s.max_heap_percent_overload) / 100,
)
