def test_clear(self): s = LimitedSet(maxlen=2) s.add('foo') s.add('bar') self.assertEqual(len(s), 2) s.clear() self.assertFalse(s)
def test_iter(self): s = LimitedSet(maxlen=2) items = 'foo', 'bar' for item in items: s.add(item) l = list(iter(s)) for item in items: self.assertIn(item, l)
def test_add_removes_duplicate_from_small_heap(self): s = LimitedSet(maxlen=2) s.add('foo') s.add('foo') s.add('foo') self.assertEqual(len(s), 1) self.assertEqual(len(s._data), 1) self.assertEqual(len(s._heap), 1)
def test_iter(self): s = LimitedSet(maxlen=2) items = "foo", "bar" for item in items: s.add(item) l = list(iter(s)) for item in items: self.assertIn(item, l)
def test_add_removes_duplicate_from_big_heap(self): s = LimitedSet(maxlen=1000) [s.add(i) for i in range(2000)] self.assertEqual(len(s), 1000) [s.add('foo') for i in range(1000)] # heap is refreshed when 15% larger than _data self.assertLess(len(s._heap), 1150) [s.add('foo') for i in range(1000)] self.assertLess(len(s._heap), 1150)
def test_iter(self): s = LimitedSet(maxlen=3) items = ['foo', 'bar', 'baz', 'xaz'] for item in items: s.add(item) l = list(iter(s)) for item in items[1:]: self.assertIn(item, l) self.assertNotIn('foo', l) self.assertListEqual(l, items[1:], 'order by insertion time')
def test_pop_and_ordering_again(self): s = LimitedSet(maxlen=5) for i in range(10): s.add(i) j = -1 for _ in range(5): i = s.pop() self.assertLess(j, i) i = s.pop() self.assertEqual(i, None)
def test_add(self): s = LimitedSet(maxlen=2) s.add("foo") s.add("bar") for n in "foo", "bar": self.assertIn(n, s) s.add("baz") for n in "bar", "baz": self.assertIn(n, s) self.assertNotIn("foo", s)
def test_add(self): s = LimitedSet(maxlen=2) s.add("foo") s.add("bar") for n in "foo", "bar": self.assertTrue(n in s) s.add("baz") for n in "bar", "baz": self.assertTrue(n in s) self.assertTrue("foo" not in s)
def test_add(self): s = LimitedSet(maxlen=2) s.add('foo') s.add('bar') for n in 'foo', 'bar': self.assertIn(n, s) s.add('baz') for n in 'bar', 'baz': self.assertIn(n, s) self.assertNotIn('foo', s)
def test_iter(self): s = LimitedSet(maxlen=3) items = ["foo", "bar", "baz", "xaz"] for item in items: s.add(item) l = list(iter(s)) for item in items[1:]: self.assertIn(item, l) self.assertNotIn("foo", l) self.assertListEqual(l, items[1:], "order by insertion time")
def test_iterable_and_ordering(self): s = LimitedSet(maxlen=35, expires=None) # we use a custom clock here, as time.time() does not have enough # precision when called quickly (can return the same value twice). clock = count(1) for i in reversed(range(15)): s.add(i, now=next(clock)) j = 40 for i in s: self.assertLess(i, j) # each item is smaller and smaller j = i self.assertEqual(i, 0) # last item is zero
def test_iter(self): if sys.platform == 'win32': raise SkipTest('Not working on Windows') s = LimitedSet(maxlen=3) items = ['foo', 'bar', 'baz', 'xaz'] for item in items: s.add(item) l = list(iter(s)) for item in items[1:]: self.assertIn(item, l) self.assertNotIn('foo', l) self.assertListEqual(l, items[1:], 'order by insertion time')
def test_discard(self): s = LimitedSet(maxlen=2) s.add('foo') s.discard('foo') self.assertNotIn('foo', s) self.assertEqual(len(s._data), 0) s.discard('foo')
def test_update(self): s1 = LimitedSet(maxlen=2) s1.add('foo') s1.add('bar') s2 = LimitedSet(maxlen=2) s2.update(s1) self.assertItemsEqual(list(s2), ['foo', 'bar']) s2.update(['bla']) self.assertItemsEqual(list(s2), ['bla', 'bar']) s2.update(['do', 're']) self.assertItemsEqual(list(s2), ['do', 're'])
def test_update(self): s1 = LimitedSet(maxlen=2) s1.add("foo") s1.add("bar") s2 = LimitedSet(maxlen=2) s2.update(s1) self.assertItemsEqual(list(s2), ["foo", "bar"]) s2.update(["bla"]) self.assertItemsEqual(list(s2), ["bla", "bar"]) s2.update(["do", "re"]) self.assertItemsEqual(list(s2), ["do", "re"])
def merge(self, d): saved = d.get('revoked') or LimitedSet() if isinstance(saved, LimitedSet): revoked.update(saved) else: # (pre 3.0.18) used to be stored as dict for item in saved: revoked.add(item) return d
def test_add(self): s = LimitedSet(maxlen=2) s.add('foo') s.add('bar') for n in 'foo', 'bar': self.assertIn(n, s) s.add('baz') for n in 'bar', 'baz': self.assertIn(n, s) self.assertNotIn('foo', s) s = LimitedSet(maxlen=10) for i in range(150): s.add(i) self.assertLessEqual(len(s), 10) # make sure heap is not leaking: self.assertLessEqual( len(s._heap), len(s) * (100. + s.max_heap_percent_overload) / 100, )
def test_start(self): try: c = Mock() c.app.connection_for_read = _amqp_connection() mingle = Mingle(c) self.assertTrue(mingle.enabled) Aig = LimitedSet() Big = LimitedSet() Aig.add("Aig-1") Aig.add("Aig-2") Big.add("Big-1") I = c.app.control.inspect.return_value = Mock() I.hello.return_value = { "*****@*****.**": {"clock": 312, "revoked": Aig._data}, "*****@*****.**": {"clock": 29, "revoked": Big._data}, "*****@*****.**": {"error": "unknown method"}, } mingle.start(c) I.hello.assert_called_with(c.hostname, worker_state.revoked._data) c.app.clock.adjust.assert_has_calls([call(312), call(29)], any_order=True) self.assertIn("Aig-1", worker_state.revoked) self.assertIn("Aig-2", worker_state.revoked) self.assertIn("Big-1", worker_state.revoked) finally: worker_state.revoked.clear()
def test_clear(self): s = LimitedSet(maxlen=2) s.add("foo") s.add("bar") self.assertEqual(len(s), 2) s.clear() self.assertFalse(s)
def test_add(self): if sys.platform == 'win32': raise SkipTest('Not working properly on Windows') s = LimitedSet(maxlen=2) s.add('foo') s.add('bar') for n in 'foo', 'bar': self.assertIn(n, s) s.add('baz') for n in 'bar', 'baz': self.assertIn(n, s) self.assertNotIn('foo', s) s = LimitedSet(maxlen=10) for i in range(150): s.add(i) self.assertLessEqual(len(s), 10) # make sure heap is not leaking: self.assertLessEqual( len(s._heap), len(s) * (100. + s.max_heap_percent_overload) / 100, )
def test_add(self): s = LimitedSet(maxlen=2) s.add("foo") s.add("bar") for n in "foo", "bar": self.assertIn(n, s) s.add("baz") for n in "bar", "baz": self.assertIn(n, s) self.assertNotIn("foo", s) s = LimitedSet(maxlen=10) for i in range(150): s.add(i) self.assertLessEqual(len(s), 10) # make sure heap is not leaking: self.assertLessEqual(len(s._heap), len(s) * (100.0 + s.max_heap_percent_overload) / 100)
def test_start(self): c = Mock() c.app.connection_for_read = _amqp_connection() mingle = Mingle(c) self.assertTrue(mingle.enabled) Aig = LimitedSet() Big = LimitedSet() Aig.add('Aig-1') Aig.add('Aig-2') Big.add('Big-1') I = c.app.control.inspect.return_value = Mock() I.hello.return_value = { '*****@*****.**': { 'clock': 312, 'revoked': Aig._data, }, '*****@*****.**': { 'clock': 29, 'revoked': Big._data, }, '*****@*****.**': { 'error': 'unknown method', }, } our_revoked = c.controller.state.revoked = LimitedSet() mingle.start(c) I.hello.assert_called_with(c.hostname, our_revoked._data) c.app.clock.adjust.assert_has_calls([ call(312), call(29), ], any_order=True) self.assertIn('Aig-1', our_revoked) self.assertIn('Aig-2', our_revoked) self.assertIn('Big-1', our_revoked)
def test_add(self): if sys.platform == 'win32': raise SkipTest('Not working properly on Windows') s = LimitedSet(maxlen=2) s.add('foo') s.add('bar') for n in 'foo', 'bar': self.assertIn(n, s) s.add('baz') for n in 'bar', 'baz': self.assertIn(n, s) self.assertNotIn('foo', s)
def test_start(self): try: c = Mock() mingle = Mingle(c) self.assertTrue(mingle.enabled) Aig = LimitedSet() Big = LimitedSet() Aig.add('Aig-1') Aig.add('Aig-2') Big.add('Big-1') I = c.app.control.inspect.return_value = Mock() I.hello.return_value = { '*****@*****.**': { 'clock': 312, 'revoked': Aig._data, }, '*****@*****.**': { 'clock': 29, 'revoked': Big._data, }, '*****@*****.**': { 'error': 'unknown method', }, } mingle.start(c) I.hello.assert_called_with() c.app.clock.adjust.assert_has_calls([ call(312), call(29), ], any_order=True) self.assertIn('Aig-1', worker_state.revoked) self.assertIn('Aig-2', worker_state.revoked) self.assertIn('Big-1', worker_state.revoked) finally: worker_state.revoked.clear()
def test_start(self): try: c = Mock() c.app.connection = _amqp_connection() mingle = Mingle(c) self.assertTrue(mingle.enabled) Aig = LimitedSet() Big = LimitedSet() Aig.add('Aig-1') Aig.add('Aig-2') Big.add('Big-1') I = c.app.control.inspect.return_value = Mock() I.hello.return_value = { '*****@*****.**': { 'clock': 312, 'revoked': Aig._data, }, '*****@*****.**': { 'clock': 29, 'revoked': Big._data, }, '*****@*****.**': { 'error': 'unknown method', }, } mingle.start(c) I.hello.assert_called_with(c.hostname, worker_state.revoked._data) c.app.clock.adjust.assert_has_calls([ call(312), call(29), ], any_order=True) self.assertIn('Aig-1', worker_state.revoked) self.assertIn('Aig-2', worker_state.revoked) self.assertIn('Big-1', worker_state.revoked) finally: worker_state.revoked.clear()
def test_as_dict(self): s = LimitedSet(maxlen=2) s.add("foo") self.assertIsInstance(s.as_dict(), dict)
def test_purge(self): # purge now enforces rules # cant purge(1) now. but .purge(now=...) still works s = LimitedSet(maxlen=10) [s.add(i) for i in range(10)] s.maxlen = 2 s.purge() self.assertEqual(len(s), 2) # expired s = LimitedSet(maxlen=10, expires=1) [s.add(i) for i in range(10)] s.maxlen = 2 s.purge(now=time() + 100) self.assertEqual(len(s), 0) # not expired s = LimitedSet(maxlen=None, expires=1) [s.add(i) for i in range(10)] s.maxlen = 2 s.purge(now=lambda: time() - 100) self.assertEqual(len(s), 2) # expired -> minsize s = LimitedSet(maxlen=10, minlen=10, expires=1) [s.add(i) for i in range(20)] s.minlen = 3 s.purge(now=time() + 3) self.assertEqual(s.minlen, len(s)) self.assertLessEqual( len(s._heap), s.maxlen * (100. + s.max_heap_percent_overload) / 100, )
def test_repr(self): s = LimitedSet(maxlen=2) items = "foo", "bar" for item in items: s.add(item) self.assertIn("LimitedSet(", repr(s))
def test_update(self): s1 = LimitedSet(maxlen=2) s1.add('foo') s1.add('bar') s2 = LimitedSet(maxlen=2) s2.update(s1) self.assertItemsEqual(list(s2), ['foo', 'bar']) s2.update(['bla']) self.assertItemsEqual(list(s2), ['bla', 'bar']) s2.update(['do', 're']) self.assertItemsEqual(list(s2), ['do', 're']) s1 = LimitedSet(maxlen=10, expires=None) s2 = LimitedSet(maxlen=10, expires=None) s3 = LimitedSet(maxlen=10, expires=None) s4 = LimitedSet(maxlen=10, expires=None) s5 = LimitedSet(maxlen=10, expires=None) for i in range(12): s1.add(i) s2.add(i*i) s3.update(s1) s3.update(s2) s4.update(s1.as_dict()) s4.update(s2.as_dict()) s5.update(s1._data) # revoke is using this s5.update(s2._data) self.assertEqual(s3, s4) self.assertEqual(s3, s5) s2.update(s4) s4.update(s2) self.assertEqual(s2, s4)
def test_update(self): s1 = LimitedSet(maxlen=2) s1.add('foo') s1.add('bar') s2 = LimitedSet(maxlen=2) s2.update(s1) self.assertItemsEqual(list(s2), ['foo', 'bar']) s2.update(['bla']) self.assertItemsEqual(list(s2), ['bla', 'bar']) s2.update(['do', 're']) self.assertItemsEqual(list(s2), ['do', 're']) s1 = LimitedSet(maxlen=10, expires=None) s2 = LimitedSet(maxlen=10, expires=None) s3 = LimitedSet(maxlen=10, expires=None) s4 = LimitedSet(maxlen=10, expires=None) s5 = LimitedSet(maxlen=10, expires=None) for i in range(12): s1.add(i) s2.add(i * i) s3.update(s1) s3.update(s2) s4.update(s1.as_dict()) s4.update(s2.as_dict()) s5.update(s1._data) # revoke is using this s5.update(s2._data) self.assertEqual(s3, s4) self.assertEqual(s3, s5) s2.update(s4) s4.update(s2) self.assertEqual(s2, s4)
def test_discard(self): s = LimitedSet(maxlen=2) s.add('foo') s.discard('foo') self.assertNotIn('foo', s) s.discard('foo')
def test_as_dict(self): s = LimitedSet(maxlen=2) s.add("foo") self.assertIsInstance(s.as_dict(), Mapping)
def test_repr(self): s = LimitedSet(maxlen=2) items = 'foo', 'bar' for item in items: s.add(item) self.assertIn('LimitedSet(', repr(s))
def test_repr(self): s = LimitedSet(maxlen=2) items = "foo", "bar" map(s.add, items) self.assertIn("LimitedSet(", repr(s))
def test_purge(self): s = LimitedSet(maxlen=None) [s.add(i) for i in range(10)] s.maxlen = 2 s.purge(1) self.assertEqual(len(s), 9) s.purge(None) self.assertEqual(len(s), 2) # expired s = LimitedSet(maxlen=None, expires=1) [s.add(i) for i in range(10)] s.maxlen = 2 s.purge(1, now=lambda: time() + 100) self.assertEqual(len(s), 9) s.purge(None, now=lambda: time() + 100) self.assertEqual(len(s), 2) # not expired s = LimitedSet(maxlen=None, expires=1) [s.add(i) for i in range(10)] s.maxlen = 2 s.purge(1, now=lambda: time() - 100) self.assertEqual(len(s), 10) s.purge(None, now=lambda: time() - 100) self.assertEqual(len(s), 10) s = LimitedSet(maxlen=None) [s.add(i) for i in range(10)] s.maxlen = 2 with patch('celery.datastructures.heappop') as hp: hp.side_effect = IndexError() s.purge() hp.assert_called_with(s._heap) with patch('celery.datastructures.heappop') as hp: s._data = {i * 2: i * 2 for i in range(10)} s.purge() self.assertEqual(hp.call_count, 10)
REVOKE_EXPIRES = 10800 #: set of all reserved :class:`~celery.worker.request.Request`'s. reserved_requests = set() #: set of currently active :class:`~celery.worker.request.Request`'s. active_requests = set() #: count of tasks accepted by the worker, sorted by type. total_count = Counter() #: count of all tasks accepted by the worker all_total_count = [0] #: the list of currently revoked tasks. Persistent if ``statedb`` set. revoked = LimitedSet(maxlen=REVOKES_MAX, expires=REVOKE_EXPIRES) #: Update global state when a task has been reserved. task_reserved = reserved_requests.add should_stop = None should_terminate = None def reset_state(): reserved_requests.clear() active_requests.clear() total_count.clear() all_total_count[:] = [0] revoked.clear()
def test_as_dict(self): s = LimitedSet(maxlen=2) s.add('foo') self.assertIsInstance(s.as_dict(), Mapping)
def add_revoked(self, *ids): for id in ids: self.p.db.setdefault('revoked', LimitedSet()).add(id)
def test_as_dict(self): s = LimitedSet(maxlen=2) s.add('foo') self.assertIsInstance(s.as_dict(), dict)
def test_pickleable(self): s = LimitedSet(maxlen=2) s.add('foo') s.add('bar') self.assertEqual(pickle.loads(pickle.dumps(s)), s)