def test_submit_bad_job_with_callback(self): """ Submission of a job that causes an exception should succeed, even with a (good) callback, but we should get a logged exception as a result. """ def _callback(future): future.result() asynchronizer = SerializingAsynchronizer( name='TestCallbackExceptionSerializingAsynchronizer', executor=self.executor, callback=_callback, ) # Submit a bad job logger_name = 'encore.concurrent.futures.abc_work_scheduler' with loghandler(logger_name) as handler: asynchronizer.submit(operator.floordiv, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, ZeroDivisionError) asynchronizer.shutdown()
def test_submit_bad_job_with_callback(self): """ Submission of a job that causes an exception should succeed, even with a (good) callback, but we should get a logged exception as a result. """ def _callback(future): future.result() asynchronizer = SerializingAsynchronizer( name='TestCallbackExceptionSerializingAsynchronizer', executor=self.executor, callback=_callback, ) # Submit a bad job logger_name = 'encore.concurrent.futures.abc_work_scheduler' with loghandler(logger_name) as handler: asynchronizer.submit(operator.div, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, ZeroDivisionError) asynchronizer.shutdown()
def test_callback(self): # Make a callback that repeats the insertion into another queue. callback_numbers = [] def _callback(future): value = future.result() callback_numbers.append(value) asynchronizer = SerializingAsynchronizer( name='TestCallbackSerializingAsynchronizer', executor=self.executor, callback=_callback ) worker1 = Worker() worker2 = Worker() asynchronizer.submit(worker1, 1) asynchronizer.submit(worker1, 2) asynchronizer.submit(worker1, 3) asynchronizer.submit(worker1, 4) asynchronizer.submit(worker1, 5) asynchronizer.submit(worker1, 6) asynchronizer.submit(worker1, 7) asynchronizer.submit(worker1, 8) asynchronizer.submit(worker1, 9) asynchronizer.submit(worker1, 10) asynchronizer.submit(worker2, 11) asynchronizer.submit(worker2, 12) asynchronizer.submit(worker2, 13) asynchronizer.submit(worker2, 14) asynchronizer.submit(worker2, 15) asynchronizer.submit(worker2, 16) asynchronizer.submit(worker2, 17) asynchronizer.submit(worker2, 18) asynchronizer.submit(worker2, 19) asynchronizer.submit(worker2, 20) asynchronizer.wait() self.assertEqual(len(worker1.data), 2) self.assertEqual(worker1.data[0], 1) self.assertEqual(worker1.data[1], 10) self.assertEqual(len(worker2.data), 1) self.assertEqual(worker2.data, [20]) self.assertEqual(len(callback_numbers), 3) self.assertEqual(callback_numbers[0], 1) self.assertEqual(callback_numbers[1], 10) self.assertEqual(callback_numbers[2], 20) asynchronizer.shutdown()
def test_callback(self): # Make a callback that repeats the insertion into another queue. callback_numbers = [] def _callback(future): value = future.result() callback_numbers.append(value) asynchronizer = SerializingAsynchronizer( name='TestCallbackSerializingAsynchronizer', executor=self.executor, callback=_callback) worker1 = Worker() worker2 = Worker() asynchronizer.submit(worker1, 1) asynchronizer.submit(worker1, 2) asynchronizer.submit(worker1, 3) asynchronizer.submit(worker1, 4) asynchronizer.submit(worker1, 5) asynchronizer.submit(worker1, 6) asynchronizer.submit(worker1, 7) asynchronizer.submit(worker1, 8) asynchronizer.submit(worker1, 9) asynchronizer.submit(worker1, 10) asynchronizer.submit(worker2, 11) asynchronizer.submit(worker2, 12) asynchronizer.submit(worker2, 13) asynchronizer.submit(worker2, 14) asynchronizer.submit(worker2, 15) asynchronizer.submit(worker2, 16) asynchronizer.submit(worker2, 17) asynchronizer.submit(worker2, 18) asynchronizer.submit(worker2, 19) asynchronizer.submit(worker2, 20) asynchronizer.wait() self.assertEqual(len(worker1.data), 2) self.assertEqual(worker1.data[0], 1) self.assertEqual(worker1.data[1], 10) self.assertEqual(len(worker2.data), 1) self.assertEqual(worker2.data, [20]) self.assertEqual(len(callback_numbers), 3) self.assertEqual(callback_numbers[0], 1) self.assertEqual(callback_numbers[1], 10) self.assertEqual(callback_numbers[2], 20) asynchronizer.shutdown()
def test_submit_job_with_raising_callback(self): """ Submission of a job with a raising callback should detect the exception in the callback. """ def _callback(future): raise TestException('Failing callback') asynchronizer = SerializingAsynchronizer( name='TestCallbackExceptionSerializingAsynchronizer', executor=self.executor, callback=_callback, ) # Submit a good job logger_name = 'encore.concurrent.futures.abc_work_scheduler' with loghandler(logger_name) as handler: asynchronizer.submit(operator.add, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, TestException) # Submit a bad job with loghandler(logger_name) as handler: asynchronizer.submit(operator.div, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, TestException) asynchronizer.shutdown()
def test_submit_job_with_raising_callback(self): """ Submission of a job with a raising callback should detect the exception in the callback. """ def _callback(future): raise TestException('Failing callback') asynchronizer = SerializingAsynchronizer( name='TestCallbackExceptionSerializingAsynchronizer', executor=self.executor, callback=_callback, ) # Submit a good job logger_name = 'encore.concurrent.futures.abc_work_scheduler' with loghandler(logger_name) as handler: asynchronizer.submit(operator.add, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, TestException) # Submit a bad job with loghandler(logger_name) as handler: asynchronizer.submit(operator.floordiv, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, TestException) asynchronizer.shutdown()
class TestSerializingAsynchronizer(unittest.TestCase): def setUp(self): self.executor = EnhancedThreadPoolExecutor( name='TestSerializingAsynchronizerExecutor', max_workers=1, ) self.asynchronizer = SerializingAsynchronizer( name='TestSerializingAsynchronizer', executor=self.executor, ) def test_events_collapsed(self): worker1 = Worker() worker2 = Worker() self.asynchronizer.submit(worker1, 1) self.asynchronizer.submit(worker1, 2) self.asynchronizer.submit(worker1, 3) self.asynchronizer.submit(worker1, 4) self.asynchronizer.submit(worker1, 5) self.asynchronizer.submit(worker1, 6) self.asynchronizer.submit(worker1, 7) self.asynchronizer.submit(worker1, 8) self.asynchronizer.submit(worker1, 9) self.asynchronizer.submit(worker1, 10) self.asynchronizer.submit(worker2, 11) self.asynchronizer.submit(worker2, 12) self.asynchronizer.submit(worker2, 13) self.asynchronizer.submit(worker2, 14) self.asynchronizer.submit(worker2, 15) self.asynchronizer.submit(worker2, 16) self.asynchronizer.submit(worker2, 17) self.asynchronizer.submit(worker2, 18) self.asynchronizer.submit(worker2, 19) self.asynchronizer.submit(worker2, 20) self.asynchronizer.wait() self.assertEqual(len(worker1.data), 2) self.assertEqual(worker1.data[0], 1) self.assertEqual(worker1.data[1], 10) self.asynchronizer.wait() self.assertEqual(len(worker2.data), 1) self.assertEqual(worker2.data, [20]) def test_callback(self): # Make a callback that repeats the insertion into another queue. callback_numbers = [] def _callback(future): value = future.result() callback_numbers.append(value) asynchronizer = SerializingAsynchronizer( name='TestCallbackSerializingAsynchronizer', executor=self.executor, callback=_callback) worker1 = Worker() worker2 = Worker() asynchronizer.submit(worker1, 1) asynchronizer.submit(worker1, 2) asynchronizer.submit(worker1, 3) asynchronizer.submit(worker1, 4) asynchronizer.submit(worker1, 5) asynchronizer.submit(worker1, 6) asynchronizer.submit(worker1, 7) asynchronizer.submit(worker1, 8) asynchronizer.submit(worker1, 9) asynchronizer.submit(worker1, 10) asynchronizer.submit(worker2, 11) asynchronizer.submit(worker2, 12) asynchronizer.submit(worker2, 13) asynchronizer.submit(worker2, 14) asynchronizer.submit(worker2, 15) asynchronizer.submit(worker2, 16) asynchronizer.submit(worker2, 17) asynchronizer.submit(worker2, 18) asynchronizer.submit(worker2, 19) asynchronizer.submit(worker2, 20) asynchronizer.wait() self.assertEqual(len(worker1.data), 2) self.assertEqual(worker1.data[0], 1) self.assertEqual(worker1.data[1], 10) self.assertEqual(len(worker2.data), 1) self.assertEqual(worker2.data, [20]) self.assertEqual(len(callback_numbers), 3) self.assertEqual(callback_numbers[0], 1) self.assertEqual(callback_numbers[1], 10) self.assertEqual(callback_numbers[2], 20) asynchronizer.shutdown() def test_asynchronizer_name(self): asynchronizer = SerializingAsynchronizer( executor=self.executor, name="Will") self.assertEqual(asynchronizer.name, "Will") self.assertEqual(asynchronizer._executor.name, 'TestSerializingAsynchronizerExecutor') def test_submit_after_shutdown(self): self.asynchronizer.shutdown() with self.assertRaises(RuntimeError): self.asynchronizer.submit(lambda: None) def test_submit_bad_job(self): """ Submission of a job that causes an exception should succeed, but we should get a logged exception as a result. """ logger_name = 'encore.concurrent.futures.abc_work_scheduler' with loghandler(logger_name) as handler: self.asynchronizer.submit(operator.floordiv, 1, 0) self.asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, ZeroDivisionError) def test_submit_bad_job_with_callback(self): """ Submission of a job that causes an exception should succeed, even with a (good) callback, but we should get a logged exception as a result. """ def _callback(future): future.result() asynchronizer = SerializingAsynchronizer( name='TestCallbackExceptionSerializingAsynchronizer', executor=self.executor, callback=_callback, ) # Submit a bad job logger_name = 'encore.concurrent.futures.abc_work_scheduler' with loghandler(logger_name) as handler: asynchronizer.submit(operator.floordiv, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, ZeroDivisionError) asynchronizer.shutdown() def test_submit_job_with_raising_callback(self): """ Submission of a job with a raising callback should detect the exception in the callback. """ def _callback(future): raise TestException('Failing callback') asynchronizer = SerializingAsynchronizer( name='TestCallbackExceptionSerializingAsynchronizer', executor=self.executor, callback=_callback, ) # Submit a good job logger_name = 'encore.concurrent.futures.abc_work_scheduler' with loghandler(logger_name) as handler: asynchronizer.submit(operator.add, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, TestException) # Submit a bad job with loghandler(logger_name) as handler: asynchronizer.submit(operator.floordiv, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, TestException) asynchronizer.shutdown() def tearDown(self): self.asynchronizer.shutdown() del self.asynchronizer self.executor.shutdown() del self.executor
class TestSerializingAsynchronizer(unittest.TestCase): def setUp(self): self.executor = EnhancedThreadPoolExecutor( name='TestSerializingAsynchronizerExecutor', max_workers=1, ) self.asynchronizer = SerializingAsynchronizer( name='TestSerializingAsynchronizer', executor=self.executor, ) def test_events_collapsed(self): worker1 = Worker() worker2 = Worker() self.asynchronizer.submit(worker1, 1) self.asynchronizer.submit(worker1, 2) self.asynchronizer.submit(worker1, 3) self.asynchronizer.submit(worker1, 4) self.asynchronizer.submit(worker1, 5) self.asynchronizer.submit(worker1, 6) self.asynchronizer.submit(worker1, 7) self.asynchronizer.submit(worker1, 8) self.asynchronizer.submit(worker1, 9) self.asynchronizer.submit(worker1, 10) self.asynchronizer.submit(worker2, 11) self.asynchronizer.submit(worker2, 12) self.asynchronizer.submit(worker2, 13) self.asynchronizer.submit(worker2, 14) self.asynchronizer.submit(worker2, 15) self.asynchronizer.submit(worker2, 16) self.asynchronizer.submit(worker2, 17) self.asynchronizer.submit(worker2, 18) self.asynchronizer.submit(worker2, 19) self.asynchronizer.submit(worker2, 20) self.asynchronizer.wait() self.assertEqual(len(worker1.data), 2) self.assertEqual(worker1.data[0], 1) self.assertEqual(worker1.data[1], 10) self.asynchronizer.wait() self.assertEqual(len(worker2.data), 1) self.assertEqual(worker2.data, [20]) def test_callback(self): # Make a callback that repeats the insertion into another queue. callback_numbers = [] def _callback(future): value = future.result() callback_numbers.append(value) asynchronizer = SerializingAsynchronizer( name='TestCallbackSerializingAsynchronizer', executor=self.executor, callback=_callback ) worker1 = Worker() worker2 = Worker() asynchronizer.submit(worker1, 1) asynchronizer.submit(worker1, 2) asynchronizer.submit(worker1, 3) asynchronizer.submit(worker1, 4) asynchronizer.submit(worker1, 5) asynchronizer.submit(worker1, 6) asynchronizer.submit(worker1, 7) asynchronizer.submit(worker1, 8) asynchronizer.submit(worker1, 9) asynchronizer.submit(worker1, 10) asynchronizer.submit(worker2, 11) asynchronizer.submit(worker2, 12) asynchronizer.submit(worker2, 13) asynchronizer.submit(worker2, 14) asynchronizer.submit(worker2, 15) asynchronizer.submit(worker2, 16) asynchronizer.submit(worker2, 17) asynchronizer.submit(worker2, 18) asynchronizer.submit(worker2, 19) asynchronizer.submit(worker2, 20) asynchronizer.wait() self.assertEqual(len(worker1.data), 2) self.assertEqual(worker1.data[0], 1) self.assertEqual(worker1.data[1], 10) self.assertEqual(len(worker2.data), 1) self.assertEqual(worker2.data, [20]) self.assertEqual(len(callback_numbers), 3) self.assertEqual(callback_numbers[0], 1) self.assertEqual(callback_numbers[1], 10) self.assertEqual(callback_numbers[2], 20) asynchronizer.shutdown() def test_asynchronizer_name(self): asynchronizer = SerializingAsynchronizer( executor=self.executor, name="Will") self.assertEqual(asynchronizer.name, "Will") self.assertEqual( asynchronizer._executor.name, 'TestSerializingAsynchronizerExecutor') def test_submit_after_shutdown(self): self.asynchronizer.shutdown() with self.assertRaises(RuntimeError): self.asynchronizer.submit(lambda: None) def test_submit_bad_job(self): """ Submission of a job that causes an exception should succeed, but we should get a logged exception as a result. """ logger_name = 'encore.concurrent.futures.abc_work_scheduler' with loghandler(logger_name) as handler: self.asynchronizer.submit(operator.div, 1, 0) self.asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, ZeroDivisionError) def test_submit_bad_job_with_callback(self): """ Submission of a job that causes an exception should succeed, even with a (good) callback, but we should get a logged exception as a result. """ def _callback(future): future.result() asynchronizer = SerializingAsynchronizer( name='TestCallbackExceptionSerializingAsynchronizer', executor=self.executor, callback=_callback, ) # Submit a bad job logger_name = 'encore.concurrent.futures.abc_work_scheduler' with loghandler(logger_name) as handler: asynchronizer.submit(operator.div, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, ZeroDivisionError) asynchronizer.shutdown() def test_submit_job_with_raising_callback(self): """ Submission of a job with a raising callback should detect the exception in the callback. """ def _callback(future): raise TestException('Failing callback') asynchronizer = SerializingAsynchronizer( name='TestCallbackExceptionSerializingAsynchronizer', executor=self.executor, callback=_callback, ) # Submit a good job logger_name = 'encore.concurrent.futures.abc_work_scheduler' with loghandler(logger_name) as handler: asynchronizer.submit(operator.add, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, TestException) # Submit a bad job with loghandler(logger_name) as handler: asynchronizer.submit(operator.div, 1, 0) asynchronizer.wait() # We log two messages for each failure. The actual traceback # from the worker, and the exception of where it occurred # (i.e. where the result was accessed) self.assertEqual(len(handler.records), 2) record = handler.records[0] self.assertIsNotNone(record.exc_info) exc_type, exc_value, exc_tb = record.exc_info self.assertIs(exc_type, TestException) asynchronizer.shutdown() def tearDown(self): self.asynchronizer.shutdown() del self.asynchronizer self.executor.shutdown() del self.executor