def test_set_wake_up_waiters(self):
count = 3
event = concurrent.ValidatingEvent()
ready = concurrent.Barrier(count + 1)
woke_up = [False] * count
def wait(n):
ready.wait(1)
woke_up[n] = event.wait(1)
threads = []
try:
for i in range(count):
t = concurrent.thread(wait, args=(i, ))
t.start()
threads.append(t)
# Wait until all threads entered the barrier.
ready.wait(1)
# Give threads time to enter the event.
time.sleep(0.5)
event.set()
finally:
for t in threads:
t.join()
self.assertTrue(all(woke_up),
"Some threads did not wake up: %s" % woke_up)
self.assertTrue(event.valid, "Event is invalid")
def test_wake_up_exactly_count_threads(self):
barrier = concurrent.Barrier(2)
lock = threading.Lock()
done = [0]
def waiter():
barrier.wait(timeout=2)
with lock:
done[0] += 1
threads = []
for i in range(3):
t = threading.Thread(target=waiter)
t.daemon = True
t.start()
threads.append(t)
try:
time.sleep(0.5)
# The first 2 threads waiting should be done now
self.assertEqual(done[0], 2)
# This should wake up the last thread waiting
barrier.wait(timeout=0)
time.sleep(0.5)
self.assertEqual(done[0], 3)
finally:
for t in threads:
t.join()
def test_max_workers_many_tasks(self):
# Check we don't get ResourceExhausted exception after reaching
# the limit on the total number of workers if TaskQueue is not full.
blocked = threading.Event()
barrier = concurrent.Barrier(self.max_workers + 1)
try:
# Exhaust workers
for i in range(self.max_workers):
task = Task(event=blocked, start_barrier=barrier)
self.executor.dispatch(task, 0)
barrier.wait(3)
# Task queue should accept further tasks up to its capacity
for i in range(self.max_tasks):
self.executor.dispatch(Task(), 0)
# Check we did what we intended -- the next task shouldn't be
# accepted
self.assertRaises(exception.ResourceExhausted,
self.executor.dispatch, Task(), 0)
finally:
# Cleanup: Finish all the executor jobs
blocked.set()
def test_invalidate_wake_up_waiters(self):
count = 3
event = concurrent.ValidatingEvent()
ready = concurrent.Barrier(count + 1)
invalidated = [False] * count
def wait(n):
ready.wait(1)
try:
event.wait(1)
except concurrent.InvalidEvent:
invalidated[n] = True
threads = []
try:
for i in range(count):
t = concurrent.thread(wait, args=(i, ))
t.start()
threads.append(t)
# Wait until all threads entered the barrier.
ready.wait(1)
# Give threads time to enter the event.
time.sleep(0.5)
event.valid = False
finally:
for t in threads:
t.join()
self.assertTrue(all(invalidated),
"Some threads were no invalidated: %s" % invalidated)
self.assertFalse(event.valid, "Event is valid")
def test_max_workers(self):
limit = self.max_workers
blocked_forever = threading.Event()
blocked = threading.Event()
try:
# Fill the executor with stuck tasks, one of them can be unblocked
# later
start_barrier = concurrent.Barrier(limit + 1)
tasks = [
Task(event=blocked_forever, start_barrier=start_barrier)
for n in range(limit - 1)
]
tasks.append(Task(event=blocked, start_barrier=start_barrier))
for t in tasks:
self.executor.dispatch(t, 0)
# Wait until all tasks are started, i.e. the executor reaches its
# maximum number of workers
start_barrier.wait(timeout=3)
# Try to run new tasks on the executor now, when the maximum number
# of workers is reached
n_extra_tasks = 2
extra_tasks = [Task() for i in range(n_extra_tasks)]
for t in extra_tasks:
self.executor.dispatch(t, 0)
# Check that none of the new tasks got executed (the number of the
# executor workers is at the maximum limit, so nothing more may be
# run)
self.assertEqual([t for t in extra_tasks if t.executed.wait(1)],
[])
# Unblock one of the tasks and check the new tasks run
blocked.set()
# The last task, the only unblocked one, should be executed now
self.assertTrue(tasks[-1].executed.wait(1))
# The other tasks shouldn't be unblocked and executed, let's check
# things go as expected before proceeding (however we don't want to
# stop and wait on each of the tasks, the first one is enough)
self.assertFalse(tasks[0].executed.wait(1))
self.assertEqual([t for t in tasks if t.executed.is_set()],
[tasks[-1]])
# Extra tasks are not blocking, they were blocked just by the
# overflown executor, so they should be all executed now when there
# is one free worker
self.assertEqual(
[t for t in extra_tasks if not t.executed.wait(1)], [])
finally:
# Cleanup: Finish all the executor jobs
blocked.set()
blocked_forever.set()
def test_thread_name(self):
thread_names = set()
barrier = concurrent.Barrier(4)
def func(x):
# Ensure that all threads are used.
barrier.wait(1)
thread_names.add(threading.current_thread().name)
list(concurrent.tmap(func, [1, 2, 3, 4], name="test"))
assert thread_names == {"test/0", "test/1", "test/2", "test/3"}
def test_no_timeout(self):
barrier = concurrent.Barrier(2)
done = threading.Event()
def waiter():
barrier.wait()
done.set()
t = threading.Thread(target=waiter)
t.daemon = True
t.start()
barrier.wait()
self.assertTrue(done.wait(timeout=0.5))
def test_worker_thread_system_name(self):
names = []
workers = 2
done = concurrent.Barrier(workers + 1)
def get_worker_name():
names.append(pthread.getname())
done.wait()
foo = executor.Executor('foo', workers, workers, None)
with utils.running(foo):
for i in range(workers):
foo.dispatch(get_worker_name)
done.wait()
self.assertEqual(sorted(names), ["foo/0", "foo/1"])
def test_block_thread(self):
barrier = concurrent.Barrier(2)
done = threading.Event()
def waiter():
barrier.wait(timeout=1)
done.set()
t = threading.Thread(target=waiter)
t.daemon = True
t.start()
try:
self.assertFalse(done.wait(timeout=0.5))
finally:
barrier.wait(timeout=0)
t.join()
def test_max_workers(self, values, max_workers, actual_workers):
workers = set()
done = threading.Event()
barrier = concurrent.Barrier(actual_workers)
def func(x):
# Ensure that all threads are used.
if not done.is_set():
barrier.wait(1)
done.set()
workers.add(threading.current_thread().ident)
list(
concurrent.tmap(func, iter(range(values)),
max_workers=max_workers))
assert len(workers) == actual_workers
def test_multiple_executors(self):
names = []
workers = 2
done = concurrent.Barrier(2 * workers + 1)
def get_worker_name():
names.append(pthread.getname())
done.wait()
foo = executor.Executor('foo', workers, workers, None)
bar = executor.Executor('bar', workers, workers, None)
with utils.running(foo), utils.running(bar):
for i in range(workers):
foo.dispatch(get_worker_name)
bar.dispatch(get_worker_name)
done.wait()
self.assertEqual(sorted(names), ["bar/0", "bar/1", "foo/0", "foo/1"])
def test_wake_up_blocked_thread(self):
barrier = concurrent.Barrier(2)
done = threading.Event()
def waiter():
barrier.wait(timeout=2)
done.set()
t = threading.Thread(target=waiter)
t.daemon = True
t.start()
try:
if done.wait(timeout=0.5):
raise RuntimeError("Thread did not block")
barrier.wait(timeout=0)
self.assertTrue(done.wait(timeout=0.5))
finally:
t.join()
def test_discarded_workers(self):
n_slow_tasks = 10
barrier = concurrent.Barrier(n_slow_tasks + 1)
slow_tasks = [Task(start_barrier=barrier) for n in range(n_slow_tasks)]
for task in slow_tasks:
self.executor.dispatch(task, 0.1)
# All workers are blocked on slow tasks
time.sleep(0.1)
# Blocked workers should be replaced with new workers
tasks = [Task(wait=0.1) for n in range(20)]
for task in tasks:
self.executor.dispatch(task, 1.0)
self.assertFalse([t for t in tasks if not task.executed.wait(1)])
barrier.wait(timeout=3)
self.assertFalse([t for t in slow_tasks if not task.executed.wait(1)])
# Discarded workers should exit, executor should operate normally
tasks = [Task(wait=0.1) for n in range(20)]
for task in tasks:
self.executor.dispatch(task, 1.0)
self.assertFalse([t for t in tasks if not task.executed.wait(1)])
def test_multiple_threads(self, count):
timeout = 5.0
# Wait for count threads + test thread
barrier = concurrent.Barrier(count + 1)
threads = []
def waiter():
time.sleep(0.1)
barrier.wait(timeout=timeout)
try:
# Start count threads waiting on the barrier
for i in range(count):
t = threading.Thread(target=waiter)
t.daemon = True
t.start()
threads.append(t)
# Wait until all threads entered the barrier. Timeout is considerd
# a failure.
barrier.wait(timeout=timeout)
finally:
for t in threads:
t.join()
def test_invalid_count(self):
with pytest.raises(ValueError):
concurrent.Barrier(0)
def test_timeout(self):
barrier = concurrent.Barrier(2)
with pytest.raises(concurrent.Timeout):
barrier.wait(0.1)
def test_last_thread(self):
barrier = concurrent.Barrier(1)
barrier.wait(timeout=0)
def test_timeout(self):
barrier = concurrent.Barrier(2)
self.assertRaises(concurrent.Timeout, barrier.wait, 0.1)
