def testSucceedNeverReraise(self): loop = global_event_loop() func = SucceedNever() func_coroutine = functools.partial(loop.run_in_executor, None, func) decorator = retry(reraise=True, try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) done, pending = loop.run_until_complete(wait([decorated_func()])) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done[0].exception(), SucceedNeverException))
def testHangForever(self): loop = global_event_loop() func = HangForever() func_coroutine = functools.partial(loop.run_in_executor, None, func) decorator = retry(try_max=2, try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) done, pending = loop.run_until_complete(wait([decorated_func()])) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done[0].exception().__cause__, TimeoutError))
def testCancelRetry(self): loop = global_event_loop() func = SucceedNever() func_coroutine = functools.partial(loop.run_in_executor, None, func) decorator = retry(try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) future = decorated_func() loop.call_later(0.3, future.cancel) done, pending = loop.run_until_complete(wait([future])) self.assertEqual(len(done), 1) self.assertTrue(done[0].cancelled())
def iter_completed(futures, max_jobs=None, max_load=None, loop=None): """ This is similar to asyncio.as_completed, but takes an iterator of futures as input, and includes support for max_jobs and max_load parameters. @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default is multiprocessing.cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is multiprocessing.cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: iterator of futures that are done @rtype: iterator """ loop = loop or global_event_loop() max_jobs = max_jobs or multiprocessing.cpu_count() max_load = max_load or multiprocessing.cpu_count() future_map = {} def task_generator(): for future in futures: future_map[id(future)] = future yield AsyncTaskFuture(future=future) scheduler = TaskScheduler( task_generator(), max_jobs=max_jobs, max_load=max_load, event_loop=loop) try: scheduler.start() # scheduler should ensure that future_map is non-empty until # task_generator is exhausted while future_map: done, pending = loop.run_until_complete( wait(*list(future_map.values()), return_when=FIRST_COMPLETED)) for future in done: del future_map[id(future)] yield future finally: # cleanup in case of interruption by SIGINT, etc scheduler.cancel() scheduler.wait()
def iter_completed(futures, max_jobs=None, max_load=None, loop=None): """ This is similar to asyncio.as_completed, but takes an iterator of futures as input, and includes support for max_jobs and max_load parameters. @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default is multiprocessing.cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is multiprocessing.cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: iterator of futures that are done @rtype: iterator """ loop = loop or global_event_loop() max_jobs = max_jobs or multiprocessing.cpu_count() max_load = max_load or multiprocessing.cpu_count() future_map = {} def task_generator(): for future in futures: future_map[id(future)] = future yield AsyncTaskFuture(future=future) scheduler = TaskScheduler(task_generator(), max_jobs=max_jobs, max_load=max_load, event_loop=loop) try: scheduler.start() # scheduler should ensure that future_map is non-empty until # task_generator is exhausted while future_map: done, pending = loop.run_until_complete( wait(list(future_map.values()), return_when=FIRST_COMPLETED)) for future in done: del future_map[id(future)] yield future finally: # cleanup in case of interruption by SIGINT, etc scheduler.cancel() scheduler.wait()
def testHangForever(self): loop = global_event_loop() func = HangForever() func_coroutine = functools.partial(loop.run_in_executor, None, func) decorator = retry(try_max=2, try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) done, pending = loop.run_until_complete(wait([decorated_func()])) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done[0].exception().__cause__, TimeoutError))
def testSucceedNeverReraise(self): loop = global_event_loop() func = SucceedNever() func_coroutine = functools.partial(loop.run_in_executor, None, func) decorator = retry(reraise=True, try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) done, pending = loop.run_until_complete(wait([decorated_func()])) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done[0].exception(), SucceedNeverException))
def testCancelRetry(self): loop = global_event_loop() func = SucceedNever() func_coroutine = functools.partial(loop.run_in_executor, None, func) decorator = retry(try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) future = decorated_func() loop.call_later(0.3, future.cancel) done, pending = loop.run_until_complete(wait([future])) self.assertEqual(len(done), 1) self.assertTrue(done[0].cancelled())