def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestSerializingAsynchronizerExecutor',
max_workers=1, )
self.asynchronizer = SerializingAsynchronizer(
name='TestSerializingAsynchronizer',
executor=self.executor, )
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestSerializerExecutor', max_workers=1)
self.serializer = Serializer(
name='TestSerializer',
executor=self.executor,
)
def test_del_shutdown(self):
executor = EnhancedThreadPoolExecutor(max_workers=5)
executor.map(abs, range(-5, 5))
threads = executor._threads
del executor
for t in threads:
t.join()
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestAsynchronizerExecutor', max_workers=1)
self.asynchronizer = DelayedAsynchronizer(
name='TestAsynchronizer',
executor=self.executor,
interval=0.5,
)
def test_del_shutdown(self):
executor = EnhancedThreadPoolExecutor(max_workers=5)
executor.map(abs, range(-5, 5))
threads = executor._threads
del executor
for t in threads:
t.join()
def test_uninitialize(self):
num_workers = 5
self.artifacts = []
def uninitialize():
self.artifacts.append(None)
executor = EnhancedThreadPoolExecutor(num_workers, uninitializer=uninitialize)
_start_all_threads(executor, num_workers)
executor.shutdown(wait=True)
self.assertEqual([None] * num_workers, self.artifacts)
def test_uninitialize(self):
num_workers = 5
self.artifacts = []
def uninitialize():
self.artifacts.append(None)
executor = EnhancedThreadPoolExecutor(num_workers, uninitializer=uninitialize)
_start_all_threads(executor, num_workers)
executor.shutdown(wait=True)
self.assertEqual([None] * num_workers, self.artifacts)
def test_context_manager_shutdown(self):
with EnhancedThreadPoolExecutor(max_workers=5) as e:
executor = e
self.assertEqual(list(e.map(abs, range(-5, 5))),
[5, 4, 3, 2, 1, 0, 1, 2, 3, 4])
for t in executor._threads:
t.join()
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestSerializerExecutor',
max_workers=1)
self.serializer = Serializer(
name='TestSerializer',
executor=self.executor,
)
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestAsynchronizerExecutor',
max_workers=1)
self.asynchronizer = Asynchronizer(
name='TestAsynchronizer',
executor=self.executor,
)
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestAsynchronizerExecutor',
max_workers=1)
self.asynchronizer = DelayedAsynchronizer(
name='TestAsynchronizer',
executor=self.executor,
interval=0.5,
)
def test_name(self):
# Unnamed executor.
executor = EnhancedThreadPoolExecutor(max_workers=5)
with executor as e:
# Start some threads.
for _ in xrange(10):
e.submit(pow, 2, 2)
expected_prefix = "{0}Worker-".format(type(e).__name__)
for t in e._threads:
self.assertTrue(t.name.startswith(expected_prefix))
# Workers should all have different names.
thread_names = [t.name for t in e._threads]
self.assertEqual(
len(set(thread_names)),
len(thread_names),
msg="Threads don't have unique names: {0}.".format(
thread_names))
# Executor with explicit name.
executor = EnhancedThreadPoolExecutor(max_workers=5,
name="DeadParrotExecutioner")
with executor as e:
self.assertEqual(e.name, "DeadParrotExecutioner")
# Start some threads.
for _ in xrange(10):
e.submit(pow, 2, 2)
expected_prefix = "DeadParrotExecutionerWorker-"
for t in e._threads:
self.assertTrue(t.name.startswith(expected_prefix))
# Workers should all have different names.
thread_names = [t.name for t in e._threads]
self.assertEqual(
len(set(thread_names)),
len(thread_names),
msg="Threads don't have unique names: {0}.".format(
thread_names))
class TestAsynchronizer(unittest.TestCase):
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestAsynchronizerExecutor',
max_workers=1)
self.asynchronizer = Asynchronizer(
name='TestAsynchronizer',
executor=self.executor,
)
def test_events_collapsed(self):
numbers = []
self.asynchronizer.submit(_worker, numbers, 1)
self.asynchronizer.submit(_worker, numbers, 2)
self.asynchronizer.submit(_worker, numbers, 3)
self.asynchronizer.submit(_worker, numbers, 4)
self.asynchronizer.submit(_worker, numbers, 5)
self.asynchronizer.submit(_worker, numbers, 6)
self.asynchronizer.submit(_worker, numbers, 7)
self.asynchronizer.submit(_worker, numbers, 8)
self.asynchronizer.submit(_worker, numbers, 9)
self.asynchronizer.submit(_worker, numbers, 10)
self.asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
def test_callback(self):
# Make a callback that repeats the insertion into another queue.
callback_numbers = []
def _callback(future):
value = future.result()
callback_numbers.append(value)
asynchronizer = Asynchronizer(
name='TestCallbackAsynchronizer',
executor=self.executor,
callback=_callback
)
numbers = []
asynchronizer.submit(_worker, numbers, 1)
asynchronizer.submit(_worker, numbers, 2)
asynchronizer.submit(_worker, numbers, 3)
asynchronizer.submit(_worker, numbers, 4)
asynchronizer.submit(_worker, numbers, 5)
asynchronizer.submit(_worker, numbers, 6)
asynchronizer.submit(_worker, numbers, 7)
asynchronizer.submit(_worker, numbers, 8)
asynchronizer.submit(_worker, numbers, 9)
asynchronizer.submit(_worker, numbers, 10)
asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
self.assertEqual(len(callback_numbers), 2)
self.assertEqual(callback_numbers[0], 1)
self.assertEqual(callback_numbers[1], 10)
asynchronizer.shutdown()
def test_asynchronizer_name(self):
asynchronizer = Asynchronizer(executor=self.executor, name="Will")
self.assertEqual(asynchronizer.name, "Will")
self.assertEqual(
asynchronizer._executor.name,
'TestAsynchronizerExecutor')
def test_submit_after_shutdown(self):
self.asynchronizer.shutdown()
with self.assertRaises(RuntimeError):
self.asynchronizer.submit(lambda: None)
def test_submit_bad_job(self):
"""
Submission of a job that causes an exception should succeed,
but we should get a logged exception as a result.
"""
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
self.asynchronizer.submit(operator.floordiv, 1, 0)
self.asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
def test_submit_bad_job_with_callback(self):
"""
Submission of a job that causes an exception should succeed,
even with a (good) callback, but we should get a logged
exception as a result.
"""
def _callback(future):
future.result()
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a bad job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.floordiv, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
asynchronizer.shutdown()
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.add, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.floordiv, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
asynchronizer.shutdown()
def tearDown(self):
self.asynchronizer.shutdown()
del self.asynchronizer
self.executor.shutdown()
del self.executor
class TestAsynchronizer(unittest.TestCase):
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestAsynchronizerExecutor', max_workers=1)
self.asynchronizer = DelayedAsynchronizer(
name='TestAsynchronizer',
executor=self.executor,
interval=0.5,
)
def tearDown(self):
self.asynchronizer.shutdown()
del self.asynchronizer
self.executor.shutdown()
del self.executor
def test_events_collapsed(self):
numbers = []
self.asynchronizer.submit(_worker, numbers, 1)
self.asynchronizer.submit(_worker, numbers, 2)
self.asynchronizer.submit(_worker, numbers, 3)
self.asynchronizer.submit(_worker, numbers, 4)
self.asynchronizer.submit(_worker, numbers, 5)
self.asynchronizer.submit(_worker, numbers, 6)
self.asynchronizer.submit(_worker, numbers, 7)
self.asynchronizer.submit(_worker, numbers, 8)
self.asynchronizer.submit(_worker, numbers, 9)
self.asynchronizer.submit(_worker, numbers, 10)
self.asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
def test_events_delayed(self):
times = []
def _worker(_list):
_list.append(time.time())
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.wait()
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.wait()
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.submit(_worker, times)
self.asynchronizer.wait()
self.assertEqual(len(times), 6)
differences = [
second - first for first, second in zip(times[:-1], times[1:])
]
for difference in differences:
self.assertGreaterEqual(difference, 0.5)
def test_callback(self):
# Make a callback that repeats the insertion into another queue.
callback_numbers = []
def _callback(future):
value = future.result()
callback_numbers.append(value)
asynchronizer = DelayedAsynchronizer(
name='TestCallbackAsynchronizer',
executor=self.executor,
interval=0.25,
callback=_callback,
)
numbers = []
asynchronizer.submit(_worker, numbers, 1)
asynchronizer.submit(_worker, numbers, 2)
asynchronizer.submit(_worker, numbers, 3)
asynchronizer.submit(_worker, numbers, 4)
asynchronizer.submit(_worker, numbers, 5)
asynchronizer.submit(_worker, numbers, 6)
asynchronizer.submit(_worker, numbers, 7)
asynchronizer.submit(_worker, numbers, 8)
asynchronizer.submit(_worker, numbers, 9)
asynchronizer.submit(_worker, numbers, 10)
asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
self.assertEqual(len(callback_numbers), 2)
self.assertEqual(callback_numbers[0], 1)
self.assertEqual(callback_numbers[1], 10)
asynchronizer.shutdown()
def test_asynchronizer_name(self):
asynchronizer = DelayedAsynchronizer(
executor=self.executor,
name="Will",
interval=0.25,
)
self.assertEqual(asynchronizer.name, "Will")
self.assertEqual(asynchronizer._executor.name,
'TestAsynchronizerExecutor')
def test_submit_after_shutdown(self):
self.asynchronizer.shutdown()
with self.assertRaises(RuntimeError):
self.asynchronizer.submit(lambda: None)
def test_submit_bad_job(self):
"""
Submission of a job that causes an exception should succeed,
but we should get a logged exception as a result.
"""
logger_name = 'encore.concurrent.futures.delayed_asynchronizer'
with loghandler(logger_name) as handler:
self.asynchronizer.submit(operator.floordiv, 1, 0)
self.asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
def test_submit_bad_job_with_callback(self):
"""
Submission of a job that causes an exception should succeed,
even with a (good) callback, but we should get a logged
exception as a result.
"""
def _callback(future):
future.result()
asynchronizer = DelayedAsynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
interval=0.25,
callback=_callback,
)
# Submit a bad job
logger_name = 'encore.concurrent.futures.delayed_asynchronizer'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.floordiv, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
asynchronizer.shutdown()
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
asynchronizer = DelayedAsynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
interval=0.25,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.delayed_asynchronizer'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.add, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.floordiv, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
asynchronizer.shutdown()
class TestSerializer(unittest.TestCase):
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestSerializerExecutor',
max_workers=1)
self.serializer = Serializer(
name='TestSerializer',
executor=self.executor,
)
def test_events_serialized(self):
numbers = []
self.serializer.submit(_worker, numbers, 1)
self.serializer.submit(_worker, numbers, 2)
self.serializer.submit(_worker, numbers, 3)
self.serializer.submit(_worker, numbers, 4)
self.serializer.submit(_worker, numbers, 5)
self.serializer.submit(_worker, numbers, 6)
self.serializer.submit(_worker, numbers, 7)
self.serializer.submit(_worker, numbers, 8)
self.serializer.submit(_worker, numbers, 9)
self.serializer.submit(_worker, numbers, 10)
self.serializer.wait()
self.assertEqual(len(numbers), 10)
self.assertEqual(numbers, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
def test_callback(self):
# Make a callback that repeats the insertion into another queue.
callback_numbers = []
def _callback(future):
value = future.result()
callback_numbers.append(value)
serializer = Serializer(
name='TestCallbackSerializer',
executor=self.executor,
callback=_callback
)
numbers = []
serializer.submit(_worker, numbers, 1)
serializer.submit(_worker, numbers, 2)
serializer.submit(_worker, numbers, 3)
serializer.submit(_worker, numbers, 4)
serializer.submit(_worker, numbers, 5)
serializer.submit(_worker, numbers, 6)
serializer.submit(_worker, numbers, 7)
serializer.submit(_worker, numbers, 8)
serializer.submit(_worker, numbers, 9)
serializer.submit(_worker, numbers, 10)
serializer.wait()
self.assertEqual(len(numbers), 10)
self.assertEqual(numbers, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
self.assertEqual(len(callback_numbers), 10)
self.assertEqual(callback_numbers, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
serializer.shutdown()
def test_serializer_name(self):
serializer = Serializer(executor=self.executor, name="Will")
self.assertEqual(serializer.name, "Will")
self.assertEqual(
serializer._executor.name,
'TestSerializerExecutor')
def test_submit_after_shutdown(self):
self.serializer.shutdown()
with self.assertRaises(RuntimeError):
self.serializer.submit(lambda: None)
def test_submit_bad_job(self):
"""
Submission of a job that causes an exception should succeed,
but we should get a logged exception as a result.
"""
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
self.serializer.submit(operator.div, 1, 0)
self.serializer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
def test_submit_bad_job_with_callback(self):
"""
Submission of a job that causes an exception should succeed,
even with a (good) callback, but we should get a logged
exception as a result.
"""
def _callback(future):
future.result()
serializer = Serializer(
name='TestCallbackExceptionSerializer',
executor=self.executor,
callback=_callback,
)
# Submit a bad job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
serializer.submit(operator.div, 1, 0)
serializer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
serializer.shutdown()
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
serializer = Serializer(
name='TestCallbackExceptionSerializer',
executor=self.executor,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
serializer.submit(operator.add, 1, 0)
serializer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
serializer.submit(operator.div, 1, 0)
serializer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
serializer.shutdown()
def tearDown(self):
self.serializer.shutdown()
del self.serializer
self.executor.shutdown()
del self.executor
class TestSerializer(unittest.TestCase):
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestSerializerExecutor',
max_workers=1)
self.serializer = Serializer(
name='TestSerializer',
executor=self.executor,
)
def test_events_serialized(self):
numbers = []
self.serializer.submit(_worker, numbers, 1)
self.serializer.submit(_worker, numbers, 2)
self.serializer.submit(_worker, numbers, 3)
self.serializer.submit(_worker, numbers, 4)
self.serializer.submit(_worker, numbers, 5)
self.serializer.submit(_worker, numbers, 6)
self.serializer.submit(_worker, numbers, 7)
self.serializer.submit(_worker, numbers, 8)
self.serializer.submit(_worker, numbers, 9)
self.serializer.submit(_worker, numbers, 10)
self.serializer.wait()
self.assertEqual(len(numbers), 10)
self.assertEqual(numbers, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
def test_callback(self):
# Make a callback that repeats the insertion into another queue.
callback_numbers = []
def _callback(future):
value = future.result()
callback_numbers.append(value)
serializer = Serializer(
name='TestCallbackSerializer',
executor=self.executor,
callback=_callback
)
numbers = []
serializer.submit(_worker, numbers, 1)
serializer.submit(_worker, numbers, 2)
serializer.submit(_worker, numbers, 3)
serializer.submit(_worker, numbers, 4)
serializer.submit(_worker, numbers, 5)
serializer.submit(_worker, numbers, 6)
serializer.submit(_worker, numbers, 7)
serializer.submit(_worker, numbers, 8)
serializer.submit(_worker, numbers, 9)
serializer.submit(_worker, numbers, 10)
serializer.wait()
self.assertEqual(len(numbers), 10)
self.assertEqual(numbers, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
self.assertEqual(len(callback_numbers), 10)
self.assertEqual(callback_numbers, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
serializer.shutdown()
def test_serializer_name(self):
serializer = Serializer(executor=self.executor, name="Will")
self.assertEqual(serializer.name, "Will")
self.assertEqual(
serializer._executor.name,
'TestSerializerExecutor')
def test_submit_after_shutdown(self):
self.serializer.shutdown()
with self.assertRaises(RuntimeError):
self.serializer.submit(lambda: None)
def test_submit_bad_job(self):
"""
Submission of a job that causes an exception should succeed,
but we should get a logged exception as a result.
"""
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
self.serializer.submit(operator.div, 1, 0)
self.serializer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
def test_submit_bad_job_with_callback(self):
"""
Submission of a job that causes an exception should succeed,
even with a (good) callback, but we should get a logged
exception as a result.
"""
def _callback(future):
future.result()
serializer = Serializer(
name='TestCallbackExceptionSerializer',
executor=self.executor,
callback=_callback,
)
# Submit a bad job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
serializer.submit(operator.div, 1, 0)
serializer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
serializer.shutdown()
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
serializer = Serializer(
name='TestCallbackExceptionSerializer',
executor=self.executor,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
serializer.submit(operator.add, 1, 0)
serializer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
serializer.submit(operator.div, 1, 0)
serializer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
serializer.shutdown()
def tearDown(self):
self.serializer.shutdown()
del self.serializer
self.executor.shutdown()
del self.executor
def __executor_default(self):
# TIP: The executor should be 'shutdown' when no longer needed to avoid
# creating multiple idle threads. This is not necessary in a small
# demo.
return EnhancedThreadPoolExecutor(max_workers=1)
class TestAsynchronizer(unittest.TestCase):
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestAsynchronizerExecutor', max_workers=1)
self.asynchronizer = Asynchronizer(
name='TestAsynchronizer',
executor=self.executor,
)
def test_events_collapsed(self):
numbers = []
self.asynchronizer.submit(_worker, numbers, 1)
self.asynchronizer.submit(_worker, numbers, 2)
self.asynchronizer.submit(_worker, numbers, 3)
self.asynchronizer.submit(_worker, numbers, 4)
self.asynchronizer.submit(_worker, numbers, 5)
self.asynchronizer.submit(_worker, numbers, 6)
self.asynchronizer.submit(_worker, numbers, 7)
self.asynchronizer.submit(_worker, numbers, 8)
self.asynchronizer.submit(_worker, numbers, 9)
self.asynchronizer.submit(_worker, numbers, 10)
self.asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
def test_callback(self):
# Make a callback that repeats the insertion into another queue.
callback_numbers = []
def _callback(future):
value = future.result()
callback_numbers.append(value)
asynchronizer = Asynchronizer(name='TestCallbackAsynchronizer',
executor=self.executor,
callback=_callback)
numbers = []
asynchronizer.submit(_worker, numbers, 1)
asynchronizer.submit(_worker, numbers, 2)
asynchronizer.submit(_worker, numbers, 3)
asynchronizer.submit(_worker, numbers, 4)
asynchronizer.submit(_worker, numbers, 5)
asynchronizer.submit(_worker, numbers, 6)
asynchronizer.submit(_worker, numbers, 7)
asynchronizer.submit(_worker, numbers, 8)
asynchronizer.submit(_worker, numbers, 9)
asynchronizer.submit(_worker, numbers, 10)
asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
self.assertEqual(len(callback_numbers), 2)
self.assertEqual(callback_numbers[0], 1)
self.assertEqual(callback_numbers[1], 10)
asynchronizer.shutdown()
def test_asynchronizer_name(self):
asynchronizer = Asynchronizer(executor=self.executor, name="Will")
self.assertEqual(asynchronizer.name, "Will")
self.assertEqual(asynchronizer._executor.name,
'TestAsynchronizerExecutor')
def test_submit_after_shutdown(self):
self.asynchronizer.shutdown()
with self.assertRaises(RuntimeError):
self.asynchronizer.submit(lambda: None)
def test_submit_bad_job(self):
"""
Submission of a job that causes an exception should succeed,
but we should get a logged exception as a result.
"""
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
self.asynchronizer.submit(operator.div, 1, 0)
self.asynchronizer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
def test_submit_bad_job_with_callback(self):
"""
Submission of a job that causes an exception should succeed,
even with a (good) callback, but we should get a logged
exception as a result.
"""
def _callback(future):
future.result()
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a bad job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.div, 1, 0)
asynchronizer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
asynchronizer.shutdown()
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.add, 1, 0)
asynchronizer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.div, 1, 0)
asynchronizer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
asynchronizer.shutdown()
def tearDown(self):
self.asynchronizer.shutdown()
del self.asynchronizer
self.executor.shutdown()
del self.executor
