def test_max_workers_many_tasks(self):
# Check we don't get TooManyTasks exception after reaching the limit on
# the total number of workers if TaskQueue is not full.
blocked = threading.Event()
barrier = concurrent.Barrier(self.max_workers + 1)
try:
# Exhaust workers
for i in range(self.max_workers):
task = Task(event=blocked, start_barrier=barrier)
self.executor.dispatch(task, 0)
barrier.wait(3)
# Task queue should accept further tasks up to its capacity
for i in range(self.max_tasks):
self.executor.dispatch(Task(), 0)
# Check we did what we intended -- the next task shouldn't be
# accepted
self.assertRaises(executor.TooManyTasks, self.executor.dispatch,
Task(), 0)
finally:
# Cleanup: Finish all the executor jobs
blocked.set()
def test_wake_up_exactly_count_threads(self):
barrier = concurrent.Barrier(2)
lock = threading.Lock()
done = [0]
def waiter():
barrier.wait(timeout=2)
with lock:
done[0] += 1
threads = []
for i in range(3):
t = threading.Thread(target=waiter)
t.daemon = True
t.start()
threads.append(t)
try:
time.sleep(0.5)
# The first 2 threads waiting should be done now
self.assertEqual(done[0], 2)
# This should wake up the last thread waiting
barrier.wait(timeout=0)
time.sleep(0.5)
self.assertEqual(done[0], 3)
finally:
for t in threads:
t.join()
def test_max_workers(self):
limit = self.max_workers
blocked_forever = threading.Event()
blocked = threading.Event()
try:
# Fill the executor with stuck tasks, one of them can be unblocked
# later
start_barrier = concurrent.Barrier(limit + 1)
tasks = [
Task(event=blocked_forever, start_barrier=start_barrier)
for n in range(limit - 1)
]
tasks.append(Task(event=blocked, start_barrier=start_barrier))
for t in tasks:
self.executor.dispatch(t, 0)
# Wait until all tasks are started, i.e. the executor reaches its
# maximum number of workers
start_barrier.wait(timeout=3)
# Try to run new tasks on the executor now, when the maximum number
# of workers is reached
n_extra_tasks = 2
extra_tasks = [Task() for i in range(n_extra_tasks)]
for t in extra_tasks:
self.executor.dispatch(t, 0)
# Check that none of the new tasks got executed (the number of the
# executor workers is at the maximum limit, so nothing more may be
# run)
self.assertEqual([t for t in extra_tasks if t.executed.wait(1)],
[])
# Unblock one of the tasks and check the new tasks run
blocked.set()
# The last task, the only unblocked one, should be executed now
self.assertTrue(tasks[-1].executed.wait(1))
# The other tasks shouldn't be unblocked and executed, let's check
# things go as expected before proceeding (however we don't want to
# stop and wait on each of the tasks, the first one is enough)
self.assertFalse(tasks[0].executed.wait(1))
self.assertEqual([t for t in tasks if t.executed.is_set()],
[tasks[-1]])
# Extra tasks are not blocking, they were blocked just by the
# overflown executor, so they should be all executed now when there
# is one free worker
self.assertEqual(
[t for t in extra_tasks if not t.executed.wait(1)], [])
finally:
# Cleanup: Finish all the executor jobs
blocked.set()
blocked_forever.set()
def test_no_timeout(self):
barrier = concurrent.Barrier(2)
done = threading.Event()
def waiter():
barrier.wait()
done.set()
t = threading.Thread(target=waiter)
t.daemon = True
t.start()
barrier.wait()
self.assertTrue(done.wait(timeout=0.5))
def test_worker_thread_system_name(self):
names = []
workers = 2
done = concurrent.Barrier(workers + 1)
def get_worker_name():
names.append(pthread.getname())
done.wait()
foo = executor.Executor('foo', workers, workers, None)
with utils.running(foo):
for i in range(workers):
foo.dispatch(get_worker_name)
done.wait()
self.assertEqual(sorted(names), ["foo/0", "foo/1"])
def test_block_thread(self):
barrier = concurrent.Barrier(2)
done = threading.Event()
def waiter():
barrier.wait(timeout=1)
done.set()
t = threading.Thread(target=waiter)
t.daemon = True
t.start()
try:
self.assertFalse(done.wait(timeout=0.5))
finally:
barrier.wait(timeout=0)
t.join()
def test_multiple_executors(self):
names = []
workers = 2
done = concurrent.Barrier(2 * workers + 1)
def get_worker_name():
names.append(pthread.getname())
done.wait()
foo = executor.Executor('foo', workers, workers, None)
bar = executor.Executor('bar', workers, workers, None)
with utils.running(foo), utils.running(bar):
for i in range(workers):
foo.dispatch(get_worker_name)
bar.dispatch(get_worker_name)
done.wait()
self.assertEqual(sorted(names), ["bar/0", "bar/1", "foo/0", "foo/1"])
def test_wake_up_blocked_thread(self):
barrier = concurrent.Barrier(2)
done = threading.Event()
def waiter():
barrier.wait(timeout=2)
done.set()
t = threading.Thread(target=waiter)
t.daemon = True
t.start()
try:
if done.wait(timeout=0.5):
raise RuntimeError("Thread did not block")
barrier.wait(timeout=0)
self.assertTrue(done.wait(timeout=0.5))
finally:
t.join()
def test_discarded_workers(self):
n_slow_tasks = 10
barrier = concurrent.Barrier(n_slow_tasks + 1)
slow_tasks = [Task(start_barrier=barrier) for n in range(n_slow_tasks)]
for task in slow_tasks:
self.executor.dispatch(task, 0.1)
# All workers are blocked on slow tasks
time.sleep(0.1)
# Blocked workers should be replaced with new workers
tasks = [Task(wait=0.1) for n in range(20)]
for task in tasks:
self.executor.dispatch(task, 1.0)
self.assertFalse([t for t in tasks if not task.executed.wait(1)])
barrier.wait(timeout=3)
self.assertFalse([t for t in slow_tasks if not task.executed.wait(1)])
# Discarded workers should exit, executor should operate normally
tasks = [Task(wait=0.1) for n in range(20)]
for task in tasks:
self.executor.dispatch(task, 1.0)
self.assertFalse([t for t in tasks if not task.executed.wait(1)])
def test_multiple_threads(self, count):
timeout = 5.0
# Wait for count threads + test thread
barrier = concurrent.Barrier(count + 1)
threads = []
def waiter():
time.sleep(0.1)
barrier.wait(timeout=timeout)
try:
# Start count threads waiting on the barrier
for i in range(count):
t = threading.Thread(target=waiter)
t.daemon = True
t.start()
threads.append(t)
# Wait until all threads entered the barrier. Timeout is considerd
# a failure.
with self.assertNotRaises():
barrier.wait(timeout=timeout)
finally:
for t in threads:
t.join()
def test_timeout(self):
barrier = concurrent.Barrier(2)
self.assertRaises(concurrent.Timeout, barrier.wait, 0.1)
def test_last_thread(self):
barrier = concurrent.Barrier(1)
barrier.wait(timeout=0)
