def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestAsynchronizerExecutor', max_workers=1)
self.asynchronizer = Asynchronizer(
name='TestAsynchronizer',
executor=self.executor,
)
def test_submit_bad_job_with_callback(self):
"""
Submission of a job that causes an exception should succeed,
even with a (good) callback, but we should get a logged
exception as a result.
"""
def _callback(future):
future.result()
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a bad job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.div, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
asynchronizer.shutdown()
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestAsynchronizerExecutor',
max_workers=1)
self.asynchronizer = Asynchronizer(
name='TestAsynchronizer',
executor=self.executor,
)
def test_submit_bad_job_with_callback(self):
"""
Submission of a job that causes an exception should succeed,
even with a (good) callback, but we should get a logged
exception as a result.
"""
def _callback(future):
future.result()
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a bad job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.floordiv, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
asynchronizer.shutdown()
def test_submit_bad_job_with_callback(self):
"""
Submission of a job that causes an exception should succeed,
even with a (good) callback, but we should get a logged
exception as a result.
"""
def _callback(future):
future.result()
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a bad job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.div, 1, 0)
asynchronizer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
asynchronizer.shutdown()
def __asynchronizer_default(self):
# TIP: Multiple Asynchronizers can share the same executor. If you
# have multiple calls that need to be "asynchronized", each should have
# its own asynchronizer.
return Asynchronizer(self._executor)
def test_callback(self):
# Make a callback that repeats the insertion into another queue.
callback_numbers = []
def _callback(future):
value = future.result()
callback_numbers.append(value)
asynchronizer = Asynchronizer(
name='TestCallbackAsynchronizer',
executor=self.executor,
callback=_callback
)
numbers = []
asynchronizer.submit(_worker, numbers, 1)
asynchronizer.submit(_worker, numbers, 2)
asynchronizer.submit(_worker, numbers, 3)
asynchronizer.submit(_worker, numbers, 4)
asynchronizer.submit(_worker, numbers, 5)
asynchronizer.submit(_worker, numbers, 6)
asynchronizer.submit(_worker, numbers, 7)
asynchronizer.submit(_worker, numbers, 8)
asynchronizer.submit(_worker, numbers, 9)
asynchronizer.submit(_worker, numbers, 10)
asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
self.assertEqual(len(callback_numbers), 2)
self.assertEqual(callback_numbers[0], 1)
self.assertEqual(callback_numbers[1], 10)
asynchronizer.shutdown()
class TestAsynchronizer(unittest.TestCase):
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestAsynchronizerExecutor',
max_workers=1)
self.asynchronizer = Asynchronizer(
name='TestAsynchronizer',
executor=self.executor,
)
def test_events_collapsed(self):
numbers = []
self.asynchronizer.submit(_worker, numbers, 1)
self.asynchronizer.submit(_worker, numbers, 2)
self.asynchronizer.submit(_worker, numbers, 3)
self.asynchronizer.submit(_worker, numbers, 4)
self.asynchronizer.submit(_worker, numbers, 5)
self.asynchronizer.submit(_worker, numbers, 6)
self.asynchronizer.submit(_worker, numbers, 7)
self.asynchronizer.submit(_worker, numbers, 8)
self.asynchronizer.submit(_worker, numbers, 9)
self.asynchronizer.submit(_worker, numbers, 10)
self.asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
def test_callback(self):
# Make a callback that repeats the insertion into another queue.
callback_numbers = []
def _callback(future):
value = future.result()
callback_numbers.append(value)
asynchronizer = Asynchronizer(
name='TestCallbackAsynchronizer',
executor=self.executor,
callback=_callback
)
numbers = []
asynchronizer.submit(_worker, numbers, 1)
asynchronizer.submit(_worker, numbers, 2)
asynchronizer.submit(_worker, numbers, 3)
asynchronizer.submit(_worker, numbers, 4)
asynchronizer.submit(_worker, numbers, 5)
asynchronizer.submit(_worker, numbers, 6)
asynchronizer.submit(_worker, numbers, 7)
asynchronizer.submit(_worker, numbers, 8)
asynchronizer.submit(_worker, numbers, 9)
asynchronizer.submit(_worker, numbers, 10)
asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
self.assertEqual(len(callback_numbers), 2)
self.assertEqual(callback_numbers[0], 1)
self.assertEqual(callback_numbers[1], 10)
asynchronizer.shutdown()
def test_asynchronizer_name(self):
asynchronizer = Asynchronizer(executor=self.executor, name="Will")
self.assertEqual(asynchronizer.name, "Will")
self.assertEqual(
asynchronizer._executor.name,
'TestAsynchronizerExecutor')
def test_submit_after_shutdown(self):
self.asynchronizer.shutdown()
with self.assertRaises(RuntimeError):
self.asynchronizer.submit(lambda: None)
def test_submit_bad_job(self):
"""
Submission of a job that causes an exception should succeed,
but we should get a logged exception as a result.
"""
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
self.asynchronizer.submit(operator.floordiv, 1, 0)
self.asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
def test_submit_bad_job_with_callback(self):
"""
Submission of a job that causes an exception should succeed,
even with a (good) callback, but we should get a logged
exception as a result.
"""
def _callback(future):
future.result()
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a bad job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.floordiv, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
asynchronizer.shutdown()
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.add, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.floordiv, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
asynchronizer.shutdown()
def tearDown(self):
self.asynchronizer.shutdown()
del self.asynchronizer
self.executor.shutdown()
del self.executor
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.add, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.floordiv, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
asynchronizer.shutdown()
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.add, 1, 0)
asynchronizer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.div, 1, 0)
asynchronizer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
asynchronizer.shutdown()
def test_callback(self):
# Make a callback that repeats the insertion into another queue.
callback_numbers = []
def _callback(future):
value = future.result()
callback_numbers.append(value)
asynchronizer = Asynchronizer(name='TestCallbackAsynchronizer',
executor=self.executor,
callback=_callback)
numbers = []
asynchronizer.submit(_worker, numbers, 1)
asynchronizer.submit(_worker, numbers, 2)
asynchronizer.submit(_worker, numbers, 3)
asynchronizer.submit(_worker, numbers, 4)
asynchronizer.submit(_worker, numbers, 5)
asynchronizer.submit(_worker, numbers, 6)
asynchronizer.submit(_worker, numbers, 7)
asynchronizer.submit(_worker, numbers, 8)
asynchronizer.submit(_worker, numbers, 9)
asynchronizer.submit(_worker, numbers, 10)
asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
self.assertEqual(len(callback_numbers), 2)
self.assertEqual(callback_numbers[0], 1)
self.assertEqual(callback_numbers[1], 10)
asynchronizer.shutdown()
class TestAsynchronizer(unittest.TestCase):
def setUp(self):
self.executor = EnhancedThreadPoolExecutor(
name='TestAsynchronizerExecutor', max_workers=1)
self.asynchronizer = Asynchronizer(
name='TestAsynchronizer',
executor=self.executor,
)
def test_events_collapsed(self):
numbers = []
self.asynchronizer.submit(_worker, numbers, 1)
self.asynchronizer.submit(_worker, numbers, 2)
self.asynchronizer.submit(_worker, numbers, 3)
self.asynchronizer.submit(_worker, numbers, 4)
self.asynchronizer.submit(_worker, numbers, 5)
self.asynchronizer.submit(_worker, numbers, 6)
self.asynchronizer.submit(_worker, numbers, 7)
self.asynchronizer.submit(_worker, numbers, 8)
self.asynchronizer.submit(_worker, numbers, 9)
self.asynchronizer.submit(_worker, numbers, 10)
self.asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
def test_callback(self):
# Make a callback that repeats the insertion into another queue.
callback_numbers = []
def _callback(future):
value = future.result()
callback_numbers.append(value)
asynchronizer = Asynchronizer(name='TestCallbackAsynchronizer',
executor=self.executor,
callback=_callback)
numbers = []
asynchronizer.submit(_worker, numbers, 1)
asynchronizer.submit(_worker, numbers, 2)
asynchronizer.submit(_worker, numbers, 3)
asynchronizer.submit(_worker, numbers, 4)
asynchronizer.submit(_worker, numbers, 5)
asynchronizer.submit(_worker, numbers, 6)
asynchronizer.submit(_worker, numbers, 7)
asynchronizer.submit(_worker, numbers, 8)
asynchronizer.submit(_worker, numbers, 9)
asynchronizer.submit(_worker, numbers, 10)
asynchronizer.wait()
self.assertEqual(len(numbers), 2)
self.assertEqual(numbers[0], 1)
self.assertEqual(numbers[1], 10)
self.assertEqual(len(callback_numbers), 2)
self.assertEqual(callback_numbers[0], 1)
self.assertEqual(callback_numbers[1], 10)
asynchronizer.shutdown()
def test_asynchronizer_name(self):
asynchronizer = Asynchronizer(executor=self.executor, name="Will")
self.assertEqual(asynchronizer.name, "Will")
self.assertEqual(asynchronizer._executor.name,
'TestAsynchronizerExecutor')
def test_submit_after_shutdown(self):
self.asynchronizer.shutdown()
with self.assertRaises(RuntimeError):
self.asynchronizer.submit(lambda: None)
def test_submit_bad_job(self):
"""
Submission of a job that causes an exception should succeed,
but we should get a logged exception as a result.
"""
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
self.asynchronizer.submit(operator.div, 1, 0)
self.asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
def test_submit_bad_job_with_callback(self):
"""
Submission of a job that causes an exception should succeed,
even with a (good) callback, but we should get a logged
exception as a result.
"""
def _callback(future):
future.result()
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a bad job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.div, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, ZeroDivisionError)
asynchronizer.shutdown()
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.add, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.div, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
asynchronizer.shutdown()
def tearDown(self):
self.asynchronizer.shutdown()
del self.asynchronizer
self.executor.shutdown()
del self.executor
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.add, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.div, 1, 0)
asynchronizer.wait()
# We log two messages for each failure. The actual traceback
# from the worker, and the exception of where it occurred
# (i.e. where the result was accessed)
self.assertEqual(len(handler.records), 2)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
asynchronizer.shutdown()
def test_asynchronizer_name(self):
asynchronizer = Asynchronizer(executor=self.executor, name="Will")
self.assertEqual(asynchronizer.name, "Will")
self.assertEqual(asynchronizer._executor.name,
'TestAsynchronizerExecutor')
def test_submit_job_with_raising_callback(self):
"""
Submission of a job with a raising callback should detect
the exception in the callback.
"""
def _callback(future):
raise _TestException('Failing callback')
asynchronizer = Asynchronizer(
name='TestCallbackExceptionAsynchronizer',
executor=self.executor,
callback=_callback,
)
# Submit a good job
logger_name = 'encore.concurrent.futures.abc_work_scheduler'
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.add, 1, 0)
asynchronizer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
# Submit a bad job
with loghandler(logger_name) as handler:
asynchronizer.submit(operator.div, 1, 0)
asynchronizer.wait()
self.assertEqual(len(handler.records), 1)
record = handler.records[0]
self.assertIsNotNone(record.exc_info)
exc_type, exc_value, exc_tb = record.exc_info
self.assertIs(exc_type, _TestException)
asynchronizer.shutdown()