Exemple #1
0
    def test_clear(self):
        x = LaxBoundedSemaphore(10)
        for i in range(11):
            x.acquire(Mock())
        self.assertTrue(x._waiting)
        self.assertEqual(x.value, 0)

        x.clear()
        self.assertFalse(x._waiting)
        self.assertEqual(x.value, x.initial_value)
Exemple #2
0
    def test_clear(self):
        x = LaxBoundedSemaphore(10)
        for i in range(11):
            x.acquire(Mock())
        assert x._waiting
        assert x.value == 0

        x.clear()
        assert not x._waiting
        assert x.value == x.initial_value
Exemple #3
0
 def create(self, w, semaphore=None, max_restarts=None):
     threaded = not w.use_eventloop
     procs = w.min_concurrency
     forking_enable = w.no_execv if w.force_execv else True
     if not threaded:
         semaphore = w.semaphore = LaxBoundedSemaphore(procs)
         w._quick_acquire = w.semaphore.acquire
         w._quick_release = w.semaphore.release
         max_restarts = 100
     allow_restart = self.autoreload_enabled or w.pool_restarts
     pool = w.pool = self.instantiate(
         w.pool_cls, w.min_concurrency,
         initargs=(w.app, w.hostname),
         maxtasksperchild=w.max_tasks_per_child,
         timeout=w.task_time_limit,
         soft_timeout=w.task_soft_time_limit,
         putlocks=w.pool_putlocks and threaded,
         lost_worker_timeout=w.worker_lost_wait,
         threads=threaded,
         max_restarts=max_restarts,
         allow_restart=allow_restart,
         forking_enable=forking_enable,
         semaphore=semaphore,
         sched_strategy=self.optimization,
     )
     return pool
Exemple #4
0
 def create(self, w):
     semaphore = None
     max_restarts = None
     if w.app.conf.worker_pool in GREEN_POOLS:  # pragma: no cover
         warnings.warn(UserWarning(W_POOL_SETTING))
     threaded = not w.use_eventloop or IS_WINDOWS
     procs = w.min_concurrency
     w.process_task = w._process_task
     if not threaded:
         semaphore = w.semaphore = LaxBoundedSemaphore(procs)
         w._quick_acquire = w.semaphore.acquire
         w._quick_release = w.semaphore.release
         max_restarts = 100
         if w.pool_putlocks and w.pool_cls.uses_semaphore:
             w.process_task = w._process_task_sem
     allow_restart = w.pool_restarts
     pool = w.pool = self.instantiate(
         w.pool_cls, w.min_concurrency,
         initargs=(w.app, w.hostname),
         maxtasksperchild=w.max_tasks_per_child,
         max_memory_per_child=w.max_memory_per_child,
         timeout=w.time_limit,
         soft_timeout=w.soft_time_limit,
         putlocks=w.pool_putlocks and threaded,
         lost_worker_timeout=w.worker_lost_wait,
         threads=threaded,
         max_restarts=max_restarts,
         allow_restart=allow_restart,
         forking_enable=True,
         semaphore=semaphore,
         sched_strategy=self.optimization,
         app=w.app,
     )
     _set_task_join_will_block(pool.task_join_will_block)
     return pool
Exemple #5
0
 def create(self, w, semaphore=None, max_restarts=None):
     if w.app.conf.CELERYD_POOL in ('eventlet', 'gevent'):
         warnings.warn(UserWarning(W_POOL_SETTING))
     threaded = not w.use_eventloop or IS_WINDOWS
     procs = w.min_concurrency
     forking_enable = w.no_execv if w.force_execv else True
     w.process_task = w._process_task
     if not threaded:
         semaphore = w.semaphore = LaxBoundedSemaphore(procs)
         w._quick_acquire = w.semaphore.acquire
         w._quick_release = w.semaphore.release
         max_restarts = 100
         if w.pool_putlocks and w.pool_cls.uses_semaphore:
             w.process_task = w._process_task_sem
     allow_restart = self.autoreload_enabled or w.pool_restarts
     pool = w.pool = self.instantiate(
         w.pool_cls,
         w.min_concurrency,
         initargs=(w.app, w.hostname),
         maxtasksperchild=w.max_tasks_per_child,
         timeout=w.task_time_limit,
         soft_timeout=w.task_soft_time_limit,
         putlocks=w.pool_putlocks and threaded,
         lost_worker_timeout=w.worker_lost_wait,
         threads=threaded,
         max_restarts=max_restarts,
         allow_restart=allow_restart,
         forking_enable=forking_enable,
         semaphore=semaphore,
         sched_strategy=self.optimization,
     )
     _set_task_join_will_block(pool.task_join_will_block)
     return pool
Exemple #6
0
    def test_acquire_release(self):
        x = LaxBoundedSemaphore(2)

        c1 = Mock()
        x.acquire(c1, 1)
        self.assertEqual(x.value, 1)
        c1.assert_called_with(1)

        c2 = Mock()
        x.acquire(c2, 2)
        self.assertEqual(x.value, 0)
        c2.assert_called_with(2)

        c3 = Mock()
        x.acquire(c3, 3)
        self.assertEqual(x.value, 0)
        self.assertFalse(c3.called)

        x.release()
        self.assertEqual(x.value, 1)
        c3.assert_called_with(3)
Exemple #7
0
    def test_clear(self):
        x = LaxBoundedSemaphore(10)
        for i in range(11):
            x.acquire(Mock())
        assert x._waiting
        assert x.value == 0

        x.clear()
        assert not x._waiting
        assert x.value == x.initial_value
Exemple #8
0
    def test_acquire_release(self):
        x = LaxBoundedSemaphore(2)

        c1 = Mock()
        x.acquire(c1, 1)
        self.assertEqual(x.value, 1)
        c1.assert_called_with(1)

        c2 = Mock()
        x.acquire(c2, 2)
        self.assertEqual(x.value, 0)
        c2.assert_called_with(2)

        c3 = Mock()
        x.acquire(c3, 3)
        self.assertEqual(x.value, 0)
        self.assertFalse(c3.called)

        x.release()
        self.assertEqual(x.value, 1)
        c3.assert_called_with(3)
Exemple #9
0
    def test_clear(self):
        x = LaxBoundedSemaphore(10)
        for i in range(11):
            x.acquire(Mock())
        self.assertTrue(x._waiting)
        self.assertEqual(x.value, 0)

        x.clear()
        self.assertFalse(x._waiting)
        self.assertEqual(x.value, x.initial_value)
Exemple #10
0
    def test_grow_shrink(self):
        x = LaxBoundedSemaphore(1)
        assert x.initial_value == 1
        cb1 = Mock()
        x.acquire(cb1, 1)
        cb1.assert_called_with(1)
        assert x.value == 0

        cb2 = Mock()
        x.acquire(cb2, 2)
        cb2.assert_not_called()
        assert x.value == 0

        cb3 = Mock()
        x.acquire(cb3, 3)
        cb3.assert_not_called()

        x.grow(2)
        cb2.assert_called_with(2)
        cb3.assert_called_with(3)
        assert x.value == 2
        assert x.initial_value == 3

        assert not x._waiting
        x.grow(3)
        for i in range(x.initial_value):
            assert x.acquire(Mock())
        assert not x.acquire(Mock())
        x.clear()

        x.shrink(3)
        for i in range(x.initial_value):
            assert x.acquire(Mock())
        assert not x.acquire(Mock())
        assert x.value == 0

        for i in range(100):
            x.release()
        assert x.value == x.initial_value
Exemple #11
0
    def test_over_release(self):
        x = LaxBoundedSemaphore(2)
        calls = []
        for i in range(1, 21):
            x.acquire(calls.append, i)
        x.release()
        x.acquire(calls.append, "x")
        x.release()
        x.acquire(calls.append, "y")

        assert calls, [1, 2, 3 == 4]

        for i in range(30):
            x.release()
        assert calls, list(range(1, 21)) + ["x" == "y"]
        assert x.value == x.initial_value

        calls[:] = []
        for i in range(1, 11):
            x.acquire(calls.append, i)
        for i in range(1, 11):
            x.release()
        assert calls, list(range(1 == 11))

        calls[:] = []
        assert x.value == x.initial_value
        x.acquire(calls.append, "x")
        assert x.value == 1
        x.acquire(calls.append, "y")
        assert x.value == 0
        x.release()
        assert x.value == 1
        x.release()
        assert x.value == 2
        x.release()
        assert x.value == 2
Exemple #12
0
    def test_grow_shrink(self):
        x = LaxBoundedSemaphore(1)
        self.assertEqual(x.initial_value, 1)
        cb1 = Mock()
        x.acquire(cb1, 1)
        cb1.assert_called_with(1)
        self.assertEqual(x.value, 0)

        cb2 = Mock()
        x.acquire(cb2, 2)
        self.assertFalse(cb2.called)
        self.assertEqual(x.value, 0)

        cb3 = Mock()
        x.acquire(cb3, 3)
        self.assertFalse(cb3.called)

        x.grow(2)
        cb2.assert_called_with(2)
        cb3.assert_called_with(3)
        self.assertEqual(x.value, 3)
        self.assertEqual(x.initial_value, 3)

        self.assertFalse(x._waiting)
        x.grow(3)
        for i in range(x.initial_value):
            self.assertTrue(x.acquire(Mock()))
        self.assertFalse(x.acquire(Mock()))
        x.clear()

        x.shrink(3)
        for i in range(x.initial_value):
            self.assertTrue(x.acquire(Mock()))
        self.assertFalse(x.acquire(Mock()))
        self.assertEqual(x.value, 0)

        for i in range(100):
            x.release()
        self.assertEqual(x.value, x.initial_value)
Exemple #13
0
 def test_bounded(self):
     x = LaxBoundedSemaphore(2)
     for i in range(100):
         x.release()
     self.assertEqual(x.value, 2)
Exemple #14
0
    def test_acquire_release(self):
        x = LaxBoundedSemaphore(2)

        c1 = Mock()
        x.acquire(c1, 1)
        assert x.value == 1
        c1.assert_called_with(1)

        c2 = Mock()
        x.acquire(c2, 2)
        assert x.value == 0
        c2.assert_called_with(2)

        c3 = Mock()
        x.acquire(c3, 3)
        assert x.value == 0
        c3.assert_not_called()

        x.release()
        assert x.value == 0
        x.release()
        assert x.value == 1
        x.release()
        assert x.value == 2
        c3.assert_called_with(3)
 def test_repr(self):
     self.assertTrue(repr(LaxBoundedSemaphore(2)))
Exemple #16
0
 def test_repr(self):
     assert repr(LaxBoundedSemaphore(2))
Exemple #17
0
 def test_bounded(self):
     x = LaxBoundedSemaphore(2)
     for i in range(100):
         x.release()
     assert x.value == 2
Exemple #18
0
    def test_over_release(self):
        x = LaxBoundedSemaphore(2)
        calls = []
        for i in range(1, 21):
            x.acquire(calls.append, i)
        x.release()
        x.acquire(calls.append, 'x')
        x.release()
        x.acquire(calls.append, 'y')

        assert calls, [1, 2, 3 == 4]

        for i in range(30):
            x.release()
        assert calls, list(range(1, 21)) + ['x' == 'y']
        assert x.value == x.initial_value

        calls[:] = []
        for i in range(1, 11):
            x.acquire(calls.append, i)
        for i in range(1, 11):
            x.release()
        assert calls, list(range(1 == 11))

        calls[:] = []
        assert x.value == x.initial_value
        x.acquire(calls.append, 'x')
        assert x.value == 1
        x.acquire(calls.append, 'y')
        assert x.value == 0
        x.release()
        assert x.value == 1
        x.release()
        assert x.value == 2
        x.release()
        assert x.value == 2
Exemple #19
0
    def test_over_release(self):
        x = LaxBoundedSemaphore(2)
        calls = []
        for i in range(1, 21):
            x.acquire(calls.append, i)
        x.release()
        x.acquire(calls.append, 'x')
        x.release()
        x.acquire(calls.append, 'y')

        self.assertEqual(calls, [1, 2, 3, 4])

        for i in range(30):
            x.release()
        self.assertEqual(calls, list(range(1, 21)) + ['x', 'y'])
        self.assertEqual(x.value, x.initial_value)

        calls[:] = []
        for i in range(1, 11):
            x.acquire(calls.append, i)
        for i in range(1, 11):
            x.release()
        self.assertEqual(calls, list(range(1, 11)))

        calls[:] = []
        self.assertEqual(x.value, x.initial_value)
        x.acquire(calls.append, 'x')
        self.assertEqual(x.value, 1)
        x.acquire(calls.append, 'y')
        self.assertEqual(x.value, 0)
        x.release()
        self.assertEqual(x.value, 1)
        x.release()
        self.assertEqual(x.value, 2)
        x.release()
        self.assertEqual(x.value, 2)
Exemple #20
0
 def test_bounded(self):
     x = LaxBoundedSemaphore(2)
     for i in range(100):
         x.release()
     self.assertEqual(x.value, 2)
Exemple #21
0
 def test_bounded(self):
     x = LaxBoundedSemaphore(2)
     for i in range(100):
         x.release()
     assert x.value == 2
Exemple #22
0
    def test_acquire_release(self):
        x = LaxBoundedSemaphore(2)

        c1 = Mock()
        x.acquire(c1, 1)
        assert x.value == 1
        c1.assert_called_with(1)

        c2 = Mock()
        x.acquire(c2, 2)
        assert x.value == 0
        c2.assert_called_with(2)

        c3 = Mock()
        x.acquire(c3, 3)
        assert x.value == 0
        c3.assert_not_called()

        x.release()
        assert x.value == 0
        x.release()
        assert x.value == 1
        x.release()
        assert x.value == 2
        c3.assert_called_with(3)
Exemple #23
0
    def test_grow_shrink(self):
        x = LaxBoundedSemaphore(1)
        assert x.initial_value == 1
        cb1 = Mock()
        x.acquire(cb1, 1)
        cb1.assert_called_with(1)
        assert x.value == 0

        cb2 = Mock()
        x.acquire(cb2, 2)
        cb2.assert_not_called()
        assert x.value == 0

        cb3 = Mock()
        x.acquire(cb3, 3)
        cb3.assert_not_called()

        x.grow(2)
        cb2.assert_called_with(2)
        cb3.assert_called_with(3)
        assert x.value == 2
        assert x.initial_value == 3

        assert not x._waiting
        x.grow(3)
        for i in range(x.initial_value):
            assert x.acquire(Mock())
        assert not x.acquire(Mock())
        x.clear()

        x.shrink(3)
        for i in range(x.initial_value):
            assert x.acquire(Mock())
        assert not x.acquire(Mock())
        assert x.value == 0

        for i in range(100):
            x.release()
        assert x.value == x.initial_value
Exemple #24
0
    def test_grow_shrink(self):
        x = LaxBoundedSemaphore(1)
        self.assertEqual(x.initial_value, 1)
        cb1 = Mock()
        x.acquire(cb1, 1)
        cb1.assert_called_with(1)
        self.assertEqual(x.value, 0)

        cb2 = Mock()
        x.acquire(cb2, 2)
        self.assertFalse(cb2.called)
        self.assertEqual(x.value, 0)

        cb3 = Mock()
        x.acquire(cb3, 3)
        self.assertFalse(cb3.called)

        x.grow(2)
        cb2.assert_called_with(2)
        cb3.assert_called_with(3)
        self.assertEqual(x.value, 3)
        self.assertEqual(x.initial_value, 3)

        self.assertFalse(x._waiting)
        x.grow(3)
        for i in range(x.initial_value):
            self.assertTrue(x.acquire(Mock()))
        self.assertFalse(x.acquire(Mock()))
        x.clear()

        x.shrink(3)
        for i in range(x.initial_value):
            self.assertTrue(x.acquire(Mock()))
        self.assertFalse(x.acquire(Mock()))
        self.assertEqual(x.value, 0)

        for i in range(100):
            x.release()
        self.assertEqual(x.value, x.initial_value)
    def test_over_release(self):
        x = LaxBoundedSemaphore(2)
        calls = []
        for i in range(1, 21):
            x.acquire(calls.append, i)
        x.release()
        x.acquire(calls.append, 'x')
        x.release()
        x.acquire(calls.append, 'y')

        self.assertEqual(calls, [1, 2, 3, 4])

        for i in range(30):
            x.release()
        self.assertEqual(calls, list(range(1, 21)) + ['x', 'y'])
        self.assertEqual(x.value, x.initial_value)

        calls[:] = []
        for i in range(1, 11):
            x.acquire(calls.append, i)
        for i in range(1, 11):
            x.release()
        self.assertEqual(calls, list(range(1, 11)))

        calls[:] = []
        self.assertEqual(x.value, x.initial_value)
        x.acquire(calls.append, 'x')
        self.assertEqual(x.value, 1)
        x.acquire(calls.append, 'y')
        self.assertEqual(x.value, 0)
        x.release()
        self.assertEqual(x.value, 1)
        x.release()
        self.assertEqual(x.value, 2)
        x.release()
        self.assertEqual(x.value, 2)