def test_max_workers_many_tasks(self): # Check we don't get TooManyTasks exception after reaching the limit on # the total number of workers if TaskQueue is not full. blocked = threading.Event() barrier = concurrent.Barrier(self.max_workers + 1) try: # Exhaust workers for i in range(self.max_workers): task = Task(event=blocked, start_barrier=barrier) self.executor.dispatch(task, 0) barrier.wait(3) # Task queue should accept further tasks up to its capacity for i in range(self.max_tasks): self.executor.dispatch(Task(), 0) # Check we did what we intended -- the next task shouldn't be # accepted self.assertRaises(executor.TooManyTasks, self.executor.dispatch, Task(), 0) finally: # Cleanup: Finish all the executor jobs blocked.set()
def test_wake_up_exactly_count_threads(self): barrier = concurrent.Barrier(2) lock = threading.Lock() done = [0] def waiter(): barrier.wait(timeout=2) with lock: done[0] += 1 threads = [] for i in range(3): t = threading.Thread(target=waiter) t.daemon = True t.start() threads.append(t) try: time.sleep(0.5) # The first 2 threads waiting should be done now self.assertEqual(done[0], 2) # This should wake up the last thread waiting barrier.wait(timeout=0) time.sleep(0.5) self.assertEqual(done[0], 3) finally: for t in threads: t.join()
def test_max_workers(self): limit = self.max_workers blocked_forever = threading.Event() blocked = threading.Event() try: # Fill the executor with stuck tasks, one of them can be unblocked # later start_barrier = concurrent.Barrier(limit + 1) tasks = [ Task(event=blocked_forever, start_barrier=start_barrier) for n in range(limit - 1) ] tasks.append(Task(event=blocked, start_barrier=start_barrier)) for t in tasks: self.executor.dispatch(t, 0) # Wait until all tasks are started, i.e. the executor reaches its # maximum number of workers start_barrier.wait(timeout=3) # Try to run new tasks on the executor now, when the maximum number # of workers is reached n_extra_tasks = 2 extra_tasks = [Task() for i in range(n_extra_tasks)] for t in extra_tasks: self.executor.dispatch(t, 0) # Check that none of the new tasks got executed (the number of the # executor workers is at the maximum limit, so nothing more may be # run) self.assertEqual([t for t in extra_tasks if t.executed.wait(1)], []) # Unblock one of the tasks and check the new tasks run blocked.set() # The last task, the only unblocked one, should be executed now self.assertTrue(tasks[-1].executed.wait(1)) # The other tasks shouldn't be unblocked and executed, let's check # things go as expected before proceeding (however we don't want to # stop and wait on each of the tasks, the first one is enough) self.assertFalse(tasks[0].executed.wait(1)) self.assertEqual([t for t in tasks if t.executed.is_set()], [tasks[-1]]) # Extra tasks are not blocking, they were blocked just by the # overflown executor, so they should be all executed now when there # is one free worker self.assertEqual( [t for t in extra_tasks if not t.executed.wait(1)], []) finally: # Cleanup: Finish all the executor jobs blocked.set() blocked_forever.set()
def test_no_timeout(self): barrier = concurrent.Barrier(2) done = threading.Event() def waiter(): barrier.wait() done.set() t = threading.Thread(target=waiter) t.daemon = True t.start() barrier.wait() self.assertTrue(done.wait(timeout=0.5))
def test_worker_thread_system_name(self): names = [] workers = 2 done = concurrent.Barrier(workers + 1) def get_worker_name(): names.append(pthread.getname()) done.wait() foo = executor.Executor('foo', workers, workers, None) with utils.running(foo): for i in range(workers): foo.dispatch(get_worker_name) done.wait() self.assertEqual(sorted(names), ["foo/0", "foo/1"])
def test_block_thread(self): barrier = concurrent.Barrier(2) done = threading.Event() def waiter(): barrier.wait(timeout=1) done.set() t = threading.Thread(target=waiter) t.daemon = True t.start() try: self.assertFalse(done.wait(timeout=0.5)) finally: barrier.wait(timeout=0) t.join()
def test_multiple_executors(self): names = [] workers = 2 done = concurrent.Barrier(2 * workers + 1) def get_worker_name(): names.append(pthread.getname()) done.wait() foo = executor.Executor('foo', workers, workers, None) bar = executor.Executor('bar', workers, workers, None) with utils.running(foo), utils.running(bar): for i in range(workers): foo.dispatch(get_worker_name) bar.dispatch(get_worker_name) done.wait() self.assertEqual(sorted(names), ["bar/0", "bar/1", "foo/0", "foo/1"])
def test_wake_up_blocked_thread(self): barrier = concurrent.Barrier(2) done = threading.Event() def waiter(): barrier.wait(timeout=2) done.set() t = threading.Thread(target=waiter) t.daemon = True t.start() try: if done.wait(timeout=0.5): raise RuntimeError("Thread did not block") barrier.wait(timeout=0) self.assertTrue(done.wait(timeout=0.5)) finally: t.join()
def test_discarded_workers(self): n_slow_tasks = 10 barrier = concurrent.Barrier(n_slow_tasks + 1) slow_tasks = [Task(start_barrier=barrier) for n in range(n_slow_tasks)] for task in slow_tasks: self.executor.dispatch(task, 0.1) # All workers are blocked on slow tasks time.sleep(0.1) # Blocked workers should be replaced with new workers tasks = [Task(wait=0.1) for n in range(20)] for task in tasks: self.executor.dispatch(task, 1.0) self.assertFalse([t for t in tasks if not task.executed.wait(1)]) barrier.wait(timeout=3) self.assertFalse([t for t in slow_tasks if not task.executed.wait(1)]) # Discarded workers should exit, executor should operate normally tasks = [Task(wait=0.1) for n in range(20)] for task in tasks: self.executor.dispatch(task, 1.0) self.assertFalse([t for t in tasks if not task.executed.wait(1)])
def test_multiple_threads(self, count): timeout = 5.0 # Wait for count threads + test thread barrier = concurrent.Barrier(count + 1) threads = [] def waiter(): time.sleep(0.1) barrier.wait(timeout=timeout) try: # Start count threads waiting on the barrier for i in range(count): t = threading.Thread(target=waiter) t.daemon = True t.start() threads.append(t) # Wait until all threads entered the barrier. Timeout is considerd # a failure. with self.assertNotRaises(): barrier.wait(timeout=timeout) finally: for t in threads: t.join()
def test_timeout(self): barrier = concurrent.Barrier(2) self.assertRaises(concurrent.Timeout, barrier.wait, 0.1)
def test_last_thread(self): barrier = concurrent.Barrier(1) barrier.wait(timeout=0)