def iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
This is similar to asyncio.as_completed, but takes an iterator of
futures as input, and includes support for max_jobs and max_load
parameters.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is multiprocessing.cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is multiprocessing.cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures that are done
@rtype: iterator
"""
loop = loop or global_event_loop()
max_jobs = max_jobs or multiprocessing.cpu_count()
max_load = max_load or multiprocessing.cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(
task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
try:
scheduler.start()
# scheduler should ensure that future_map is non-empty until
# task_generator is exhausted
while future_map:
done, pending = loop.run_until_complete(
wait(*list(future_map.values()), return_when=FIRST_COMPLETED))
for future in done:
del future_map[id(future)]
yield future
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
This is similar to asyncio.as_completed, but takes an iterator of
futures as input, and includes support for max_jobs and max_load
parameters.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is multiprocessing.cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is multiprocessing.cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures that are done
@rtype: iterator
"""
loop = loop or global_event_loop()
max_jobs = max_jobs or multiprocessing.cpu_count()
max_load = max_load or multiprocessing.cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
try:
scheduler.start()
# scheduler should ensure that future_map is non-empty until
# task_generator is exhausted
while future_map:
done, pending = loop.run_until_complete(
wait(list(future_map.values()), return_when=FIRST_COMPLETED))
for future in done:
del future_map[id(future)]
yield future
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
An asynchronous version of iter_completed. This yields futures, which
when done, result in a set of input futures that are done. This serves
as a wrapper around portage's internal TaskScheduler class, using
standard asyncio interfaces.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures, which when done, result in a set of
input futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
max_jobs = max_jobs or get_cpu_count()
max_load = max_load or get_cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""
if future_done_set.cancelled():
return
done, pending = wait_result.result()
for future in done:
del future_map[id(future)]
future_done_set.set_result(done)
def cancel_callback(wait_result, future_done_set):
"""Cancel wait_result if future_done_set has been cancelled."""
if future_done_set.cancelled() and not wait_result.done():
wait_result.cancel()
try:
scheduler.start()
# scheduler should ensure that future_map is non-empty until
# task_generator is exhausted
while future_map:
wait_result = asyncio.ensure_future(asyncio.wait(
list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED,
loop=loop),
loop=loop)
future_done_set = loop.create_future()
future_done_set.add_done_callback(
functools.partial(cancel_callback, wait_result))
wait_result.add_done_callback(
functools.partial(done_callback, future_done_set))
yield future_done_set
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
An asynchronous version of iter_completed. This yields futures, which
when done, result in a set of input futures that are done. This serves
as a wrapper around portage's internal TaskScheduler class, using
standard asyncio interfaces.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures, which when done, result in a set of
input futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
max_jobs = max_jobs or get_cpu_count()
max_load = max_load or get_cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""
if future_done_set.cancelled():
return
done, pending = wait_result.result()
for future in done:
del future_map[id(future)]
future_done_set.set_result(done)
def cancel_callback(wait_result, future_done_set):
"""Cancel wait_result if future_done_set has been cancelled."""
if future_done_set.cancelled() and not wait_result.done():
wait_result.cancel()
@coroutine
def fetch_wait_result(scheduler, first, loop=None):
if first:
yield scheduler.async_start()
# If the current coroutine awakens just after a call to
# done_callback but before scheduler has been notified of
# corresponding done future(s), then wait here until scheduler
# is notified (which will cause future_map to populate).
while not future_map and scheduler.poll() is None:
yield asyncio.sleep(0, loop=loop)
if not future_map:
if scheduler.poll() is not None:
coroutine_return((set(), set()))
else:
raise AssertionError('expected non-empty future_map')
wait_result = yield asyncio.wait(list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED,
loop=loop)
coroutine_return(wait_result)
first = True
try:
while True:
wait_result = asyncio.ensure_future(fetch_wait_result(scheduler,
first,
loop=loop),
loop=loop)
first = False
future_done_set = loop.create_future()
future_done_set.add_done_callback(
functools.partial(cancel_callback, wait_result))
wait_result.add_done_callback(
functools.partial(done_callback, future_done_set))
yield future_done_set
if not future_map and scheduler.poll() is not None:
break
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
An asynchronous version of iter_completed. This yields futures, which
when done, result in a set of input futures that are done. This serves
as a wrapper around portage's internal TaskScheduler class, using
standard asyncio interfaces.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures, which when done, result in a set of
input futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
max_jobs = max_jobs or get_cpu_count()
max_load = max_load or get_cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(
task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""
if future_done_set.cancelled():
return
done, pending = wait_result.result()
for future in done:
del future_map[id(future)]
future_done_set.set_result(done)
def cancel_callback(wait_result, future_done_set):
"""Cancel wait_result if future_done_set has been cancelled."""
if future_done_set.cancelled() and not wait_result.done():
wait_result.cancel()
try:
scheduler.start()
# scheduler should ensure that future_map is non-empty until
# task_generator is exhausted
while future_map:
wait_result = asyncio.ensure_future(
asyncio.wait(list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop)
future_done_set = loop.create_future()
future_done_set.add_done_callback(
functools.partial(cancel_callback, wait_result))
wait_result.add_done_callback(
functools.partial(done_callback, future_done_set))
yield future_done_set
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()