def testAsyncCancel(self): loop = global_event_loop() input_futures = set() future_count = 3 def future_generator(): for i in range(future_count): future = loop.create_future() loop.call_soon(lambda future: None if future.done() else future.set_result(None), future) input_futures.add(future) yield future for future_done_set in async_iter_completed(future_generator(), max_jobs=True, max_load=True, loop=loop): future_done_set.cancel() break # With max_jobs=True, async_iter_completed should have executed # the generator until it raised StopIteration. self.assertEqual(future_count, len(input_futures)) loop.run_until_complete(asyncio.wait(input_futures, loop=loop)) # The futures may have results or they may have been cancelled # by TaskScheduler, and behavior varies depending on the python # interpreter. for future in input_futures: future.cancelled() or future.result()
def _testPipeLoggerToPipe(self, test_string, loop=None):
"""
Test PipeLogger writing to a pipe connected to a PipeReader.
This verifies that PipeLogger does not deadlock when writing
to a pipe that's drained by a PipeReader running in the same
process (requires non-blocking write).
"""
input_fd, writer_pipe = os.pipe()
_set_nonblocking(writer_pipe)
writer_pipe = os.fdopen(writer_pipe, 'wb', 0)
writer = asyncio.ensure_future(_writer(writer_pipe, test_string.encode('ascii'), loop=loop), loop=loop)
writer.add_done_callback(lambda writer: writer_pipe.close())
pr, pw = os.pipe()
consumer = PipeLogger(background=True,
input_fd=input_fd,
log_file_path=os.fdopen(pw, 'wb', 0),
scheduler=loop)
consumer.start()
# Before starting the reader, wait here for a moment, in order
# to exercise PipeLogger's handling of EAGAIN during write.
yield asyncio.wait([writer], timeout=0.01)
reader = _reader(pr, loop=loop)
yield writer
content = yield reader
yield consumer.async_wait()
self.assertEqual(consumer.returncode, os.EX_OK)
coroutine_return(content.decode('ascii', 'replace'))
def testHangForeverReraise(self): loop = global_event_loop() func_coroutine = self._wrap_coroutine_func(HangForever()) decorator = retry(reraise=True, try_max=2, try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception(), asyncio.TimeoutError))
def testSucceedNever(self): loop = global_event_loop() func_coroutine = self._wrap_coroutine_func(SucceedNever()) decorator = retry(try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception().__cause__, SucceedNeverException))
def testOverallTimeoutWithTimeoutError(self): loop = global_event_loop() # results in TimeoutError because it hangs forever func_coroutine = self._wrap_coroutine_func(HangForever()) decorator = retry(try_timeout=0.1, overall_timeout=0.3, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception().__cause__, asyncio.TimeoutError))
def testCancelRetry(self): loop = global_event_loop() func_coroutine = self._wrap_coroutine_func(SucceedNever()) decorator = retry(try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine, loop=loop) future = decorated_func() loop.call_later(0.3, future.cancel) done, pending = loop.run_until_complete(asyncio.wait([future], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(done.pop().cancelled())
def testCancelRetry(self):
loop = global_event_loop()
func_coroutine = self._wrap_coroutine_func(SucceedNever())
decorator = retry(try_timeout=0.1,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine, loop=loop)
future = decorated_func()
loop.call_later(0.3, future.cancel)
done, pending = loop.run_until_complete(
asyncio.wait([future], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(done.pop().cancelled())
def testHangForever(self):
loop = global_event_loop()
func_coroutine = self._wrap_coroutine_func(HangForever())
decorator = retry(try_max=2,
try_timeout=0.1,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine, loop=loop)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception().__cause__, asyncio.TimeoutError))
def _wait_loop(self, timeout=None): loop = self.scheduler tasks = [self.async_wait()] if timeout is not None: tasks.append(asyncio.ensure_future( asyncio.sleep(timeout, loop=loop), loop=loop)) try: loop.run_until_complete(asyncio.ensure_future( asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop)) finally: for task in tasks: task.cancel()
def _wait_loop(self, timeout=None): loop = self.scheduler tasks = [self.async_wait()] if timeout is not None: tasks.append(asyncio.ensure_future( asyncio.sleep(timeout, loop=loop), loop=loop)) try: loop.run_until_complete(asyncio.ensure_future( asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop)) finally: for task in tasks: task.cancel()
def testOverallTimeoutWithException(self):
loop = global_event_loop()
func_coroutine = self._wrap_coroutine_func(SucceedNever())
decorator = retry(try_timeout=0.1,
overall_timeout=0.3,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine, loop=loop)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception().__cause__,
SucceedNeverException))
def testSucceedNeverReraise(self):
loop = global_event_loop()
with self._wrap_coroutine_func(SucceedNever()) as func_coroutine:
decorator = retry(reraise=True,
try_max=4,
try_timeout=None,
delay_func=RandomExponentialBackoff(
multiplier=0.1, base=2))
decorated_func = decorator(func_coroutine, loop=loop)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception(), SucceedNeverException))
def testSucceedNever(self):
loop = global_event_loop()._asyncio_wrapper
func = SucceedNever()
func_coroutine = functools.partial(loop.run_in_executor, None, func)
decorator = retry(try_max=4,
try_timeout=None,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception().__cause__,
SucceedNeverException))
def testOverallTimeoutWithTimeoutError(self):
loop = global_event_loop()._asyncio_wrapper
# results in TimeoutError because it hangs forever
func = HangForever()
func_coroutine = functools.partial(loop.run_in_executor, None, func)
decorator = retry(try_timeout=0.1,
overall_timeout=0.3,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception().__cause__, asyncio.TimeoutError))
def testHangForeverReraise(self):
loop = global_event_loop()._asyncio_wrapper
func = HangForever()
func_coroutine = functools.partial(loop.run_in_executor, None, func)
decorator = retry(reraise=True,
try_max=2,
try_timeout=0.1,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception(), asyncio.TimeoutError))
def test_cancelled_future(self):
"""
When a coroutine raises CancelledError, the coroutine's
future is cancelled.
"""
@coroutine
def cancelled_future_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
while True:
future = loop.create_future()
loop.call_soon(future.cancel)
yield future
loop = asyncio.get_event_loop()
future = loop.run_until_complete(
asyncio.wait([cancelled_future_coroutine()]))[0].pop()
self.assertTrue(future.cancelled())
def test_cancelled_future(self): """ When a coroutine raises CancelledError, the coroutine's future is cancelled. """ @coroutine def cancelled_future_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: future = loop.create_future() loop.call_soon(future.cancel) yield future loop = asyncio.get_event_loop() future = loop.run_until_complete(asyncio.wait([cancelled_future_coroutine()]))[0].pop() self.assertTrue(future.cancelled())
def fetch_wait_result(scheduler, first, loop=None):
if first:
yield scheduler.async_start()
# If the current coroutine awakens just after a call to
# done_callback but before scheduler has been notified of
# corresponding done future(s), then wait here until scheduler
# is notified (which will cause future_map to populate).
while not future_map and scheduler.poll() is None:
yield asyncio.sleep(0, loop=loop)
if not future_map:
if scheduler.poll() is not None:
coroutine_return((set(), set()))
else:
raise AssertionError('expected non-empty future_map')
wait_result = yield asyncio.wait(list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED,
loop=loop)
coroutine_return(wait_result)
def test_method_coroutine(self):
class Cubby:
_empty = object()
def __init__(self, loop):
self._loop = loop
self._value = self._empty
self._waiters = []
def _notify(self):
waiters = self._waiters
self._waiters = []
for waiter in waiters:
waiter.cancelled() or waiter.set_result(None)
def _wait(self):
waiter = self._loop.create_future()
self._waiters.append(waiter)
return waiter
@coroutine
def read(self, loop=None):
while self._value is self._empty:
yield self._wait()
value = self._value
self._value = self._empty
self._notify()
coroutine_return(value)
@coroutine
def write(self, value, loop=None):
while self._value is not self._empty:
yield self._wait()
self._value = value
self._notify()
@coroutine
def writer_coroutine(cubby, values, sentinel, loop=None):
for value in values:
yield cubby.write(value, loop=loop)
yield cubby.write(sentinel, loop=loop)
@coroutine
def reader_coroutine(cubby, sentinel, loop=None):
results = []
while True:
result = yield cubby.read(loop=loop)
if result == sentinel:
break
results.append(result)
coroutine_return(results)
loop = asyncio.get_event_loop()
cubby = Cubby(loop)
values = list(range(3))
writer = asyncio.ensure_future(writer_coroutine(cubby,
values,
None,
loop=loop),
loop=loop)
reader = asyncio.ensure_future(reader_coroutine(cubby, None,
loop=loop),
loop=loop)
loop.run_until_complete(asyncio.wait([writer, reader], loop=loop))
self.assertEqual(reader.result(), values)
# Test decoration of coroutine methods and functions for
# synchronous usage, allowing coroutines to smoothly
# blend with synchronous code.
sync_cubby = _sync_methods(cubby, loop=loop)
sync_reader = _sync_decorator(reader_coroutine, loop=loop)
writer = asyncio.ensure_future(writer_coroutine(cubby,
values,
None,
loop=loop),
loop=loop)
self.assertEqual(sync_reader(cubby, None), values)
self.assertTrue(writer.done())
for i in range(3):
sync_cubby.write(i)
self.assertEqual(sync_cubby.read(), i)
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
An asynchronous version of iter_completed. This yields futures, which
when done, result in a set of input futures that are done. This serves
as a wrapper around portage's internal TaskScheduler class, using
standard asyncio interfaces.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures, which when done, result in a set of
input futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
max_jobs = max_jobs or get_cpu_count()
max_load = max_load or get_cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(
task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""
if future_done_set.cancelled():
return
done, pending = wait_result.result()
for future in done:
del future_map[id(future)]
future_done_set.set_result(done)
def cancel_callback(wait_result, future_done_set):
"""Cancel wait_result if future_done_set has been cancelled."""
if future_done_set.cancelled() and not wait_result.done():
wait_result.cancel()
try:
scheduler.start()
# scheduler should ensure that future_map is non-empty until
# task_generator is exhausted
while future_map:
wait_result = asyncio.ensure_future(
asyncio.wait(list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop)
future_done_set = loop.create_future()
future_done_set.add_done_callback(
functools.partial(cancel_callback, wait_result))
wait_result.add_done_callback(
functools.partial(done_callback, future_done_set))
yield future_done_set
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def test_method_coroutine(self): class Cubby(object): _empty = object() def __init__(self, loop): self._loop = loop self._value = self._empty self._waiters = [] def _notify(self): waiters = self._waiters self._waiters = [] for waiter in waiters: waiter.cancelled() or waiter.set_result(None) def _wait(self): waiter = self._loop.create_future() self._waiters.append(waiter) return waiter @coroutine def read(self): while self._value is self._empty: yield self._wait() value = self._value self._value = self._empty self._notify() coroutine_return(value) @coroutine def write(self, value): while self._value is not self._empty: yield self._wait() self._value = value self._notify() @coroutine def writer_coroutine(cubby, values, sentinel): for value in values: yield cubby.write(value) yield cubby.write(sentinel) @coroutine def reader_coroutine(cubby, sentinel): results = [] while True: result = yield cubby.read() if result == sentinel: break results.append(result) coroutine_return(results) loop = asyncio.get_event_loop() cubby = Cubby(loop) values = list(range(3)) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) reader = asyncio.ensure_future(reader_coroutine(cubby, None), loop=loop) loop.run_until_complete(asyncio.wait([writer, reader])) self.assertEqual(reader.result(), values)
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
An asynchronous version of iter_completed. This yields futures, which
when done, result in a set of input futures that are done. This serves
as a wrapper around portage's internal TaskScheduler class, using
standard asyncio interfaces.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures, which when done, result in a set of
input futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
max_jobs = max_jobs or get_cpu_count()
max_load = max_load or get_cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""
if future_done_set.cancelled():
return
done, pending = wait_result.result()
for future in done:
del future_map[id(future)]
future_done_set.set_result(done)
def cancel_callback(wait_result, future_done_set):
"""Cancel wait_result if future_done_set has been cancelled."""
if future_done_set.cancelled() and not wait_result.done():
wait_result.cancel()
try:
scheduler.start()
# scheduler should ensure that future_map is non-empty until
# task_generator is exhausted
while future_map:
wait_result = asyncio.ensure_future(asyncio.wait(
list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED,
loop=loop),
loop=loop)
future_done_set = loop.create_future()
future_done_set.add_done_callback(
functools.partial(cancel_callback, wait_result))
wait_result.add_done_callback(
functools.partial(done_callback, future_done_set))
yield future_done_set
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def test_method_coroutine(self): class Cubby(object): _empty = object() def __init__(self, loop): self._loop = loop self._value = self._empty self._waiters = [] def _notify(self): waiters = self._waiters self._waiters = [] for waiter in waiters: waiter.cancelled() or waiter.set_result(None) def _wait(self): waiter = self._loop.create_future() self._waiters.append(waiter) return waiter @coroutine def read(self): while self._value is self._empty: yield self._wait() value = self._value self._value = self._empty self._notify() coroutine_return(value) @coroutine def write(self, value): while self._value is not self._empty: yield self._wait() self._value = value self._notify() @coroutine def writer_coroutine(cubby, values, sentinel): for value in values: yield cubby.write(value) yield cubby.write(sentinel) @coroutine def reader_coroutine(cubby, sentinel): results = [] while True: result = yield cubby.read() if result == sentinel: break results.append(result) coroutine_return(results) loop = asyncio.get_event_loop() cubby = Cubby(loop) values = list(range(3)) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) reader = asyncio.ensure_future(reader_coroutine(cubby, None), loop=loop) loop.run_until_complete(asyncio.wait([writer, reader])) self.assertEqual(reader.result(), values) # Test decoration of coroutine methods and functions for # synchronous usage, allowing coroutines to smoothly # blend with synchronous code. sync_cubby = _sync_methods(cubby, loop=loop) sync_reader = _sync_decorator(reader_coroutine, loop=loop) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) self.assertEqual(sync_reader(cubby, None), values) self.assertTrue(writer.done()) for i in range(3): sync_cubby.write(i) self.assertEqual(sync_cubby.read(), i)