Example #1
0
def iter_completed(futures, max_jobs=None, max_load=None, loop=None):
	"""
	This is similar to asyncio.as_completed, but takes an iterator of
	futures as input, and includes support for max_jobs and max_load
	parameters.

	@param futures: iterator of asyncio.Future (or compatible)
	@type futures: iterator
	@param max_jobs: max number of futures to process concurrently (default
		is multiprocessing.cpu_count())
	@type max_jobs: int
	@param max_load: max load allowed when scheduling a new future,
		otherwise schedule no more than 1 future at a time (default
		is multiprocessing.cpu_count())
	@type max_load: int or float
	@param loop: event loop
	@type loop: EventLoop
	@return: iterator of futures that are done
	@rtype: iterator
	"""
	loop = loop or global_event_loop()
	max_jobs = max_jobs or multiprocessing.cpu_count()
	max_load = max_load or multiprocessing.cpu_count()

	future_map = {}
	def task_generator():
		for future in futures:
			future_map[id(future)] = future
			yield AsyncTaskFuture(future=future)

	scheduler = TaskScheduler(
		task_generator(),
		max_jobs=max_jobs,
		max_load=max_load,
		event_loop=loop)

	try:
		scheduler.start()

		# scheduler should ensure that future_map is non-empty until
		# task_generator is exhausted
		while future_map:
			done, pending = loop.run_until_complete(
				wait(*list(future_map.values()), return_when=FIRST_COMPLETED))
			for future in done:
				del future_map[id(future)]
				yield future

	finally:
		# cleanup in case of interruption by SIGINT, etc
		scheduler.cancel()
		scheduler.wait()
Example #2
0
def iter_completed(futures, max_jobs=None, max_load=None, loop=None):
    """
	This is similar to asyncio.as_completed, but takes an iterator of
	futures as input, and includes support for max_jobs and max_load
	parameters.

	@param futures: iterator of asyncio.Future (or compatible)
	@type futures: iterator
	@param max_jobs: max number of futures to process concurrently (default
		is multiprocessing.cpu_count())
	@type max_jobs: int
	@param max_load: max load allowed when scheduling a new future,
		otherwise schedule no more than 1 future at a time (default
		is multiprocessing.cpu_count())
	@type max_load: int or float
	@param loop: event loop
	@type loop: EventLoop
	@return: iterator of futures that are done
	@rtype: iterator
	"""
    loop = loop or global_event_loop()
    max_jobs = max_jobs or multiprocessing.cpu_count()
    max_load = max_load or multiprocessing.cpu_count()

    future_map = {}

    def task_generator():
        for future in futures:
            future_map[id(future)] = future
            yield AsyncTaskFuture(future=future)

    scheduler = TaskScheduler(task_generator(),
                              max_jobs=max_jobs,
                              max_load=max_load,
                              event_loop=loop)

    try:
        scheduler.start()

        # scheduler should ensure that future_map is non-empty until
        # task_generator is exhausted
        while future_map:
            done, pending = loop.run_until_complete(
                wait(list(future_map.values()), return_when=FIRST_COMPLETED))
            for future in done:
                del future_map[id(future)]
                yield future

    finally:
        # cleanup in case of interruption by SIGINT, etc
        scheduler.cancel()
        scheduler.wait()
Example #3
0
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
    """
	An asynchronous version of iter_completed. This yields futures, which
	when done, result in a set of input futures that are done. This serves
	as a wrapper around portage's internal TaskScheduler class, using
	standard asyncio interfaces.

	@param futures: iterator of asyncio.Future (or compatible)
	@type futures: iterator
	@param max_jobs: max number of futures to process concurrently (default
		is portage.util.cpuinfo.get_cpu_count())
	@type max_jobs: int
	@param max_load: max load allowed when scheduling a new future,
		otherwise schedule no more than 1 future at a time (default
		is portage.util.cpuinfo.get_cpu_count())
	@type max_load: int or float
	@param loop: event loop
	@type loop: EventLoop
	@return: iterator of futures, which when done, result in a set of
		input futures that are done
	@rtype: iterator
	"""
    loop = asyncio._wrap_loop(loop)

    max_jobs = max_jobs or get_cpu_count()
    max_load = max_load or get_cpu_count()

    future_map = {}

    def task_generator():
        for future in futures:
            future_map[id(future)] = future
            yield AsyncTaskFuture(future=future)

    scheduler = TaskScheduler(task_generator(),
                              max_jobs=max_jobs,
                              max_load=max_load,
                              event_loop=loop)

    def done_callback(future_done_set, wait_result):
        """Propagate results from wait_result to future_done_set."""
        if future_done_set.cancelled():
            return
        done, pending = wait_result.result()
        for future in done:
            del future_map[id(future)]
        future_done_set.set_result(done)

    def cancel_callback(wait_result, future_done_set):
        """Cancel wait_result if future_done_set has been cancelled."""
        if future_done_set.cancelled() and not wait_result.done():
            wait_result.cancel()

    try:
        scheduler.start()

        # scheduler should ensure that future_map is non-empty until
        # task_generator is exhausted
        while future_map:
            wait_result = asyncio.ensure_future(asyncio.wait(
                list(future_map.values()),
                return_when=asyncio.FIRST_COMPLETED,
                loop=loop),
                                                loop=loop)
            future_done_set = loop.create_future()
            future_done_set.add_done_callback(
                functools.partial(cancel_callback, wait_result))
            wait_result.add_done_callback(
                functools.partial(done_callback, future_done_set))
            yield future_done_set
    finally:
        # cleanup in case of interruption by SIGINT, etc
        scheduler.cancel()
        scheduler.wait()
Example #4
0
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
    """
	An asynchronous version of iter_completed. This yields futures, which
	when done, result in a set of input futures that are done. This serves
	as a wrapper around portage's internal TaskScheduler class, using
	standard asyncio interfaces.

	@param futures: iterator of asyncio.Future (or compatible)
	@type futures: iterator
	@param max_jobs: max number of futures to process concurrently (default
		is portage.util.cpuinfo.get_cpu_count())
	@type max_jobs: int
	@param max_load: max load allowed when scheduling a new future,
		otherwise schedule no more than 1 future at a time (default
		is portage.util.cpuinfo.get_cpu_count())
	@type max_load: int or float
	@param loop: event loop
	@type loop: EventLoop
	@return: iterator of futures, which when done, result in a set of
		input futures that are done
	@rtype: iterator
	"""
    loop = asyncio._wrap_loop(loop)

    max_jobs = max_jobs or get_cpu_count()
    max_load = max_load or get_cpu_count()

    future_map = {}

    def task_generator():
        for future in futures:
            future_map[id(future)] = future
            yield AsyncTaskFuture(future=future)

    scheduler = TaskScheduler(task_generator(),
                              max_jobs=max_jobs,
                              max_load=max_load,
                              event_loop=loop)

    def done_callback(future_done_set, wait_result):
        """Propagate results from wait_result to future_done_set."""
        if future_done_set.cancelled():
            return
        done, pending = wait_result.result()
        for future in done:
            del future_map[id(future)]
        future_done_set.set_result(done)

    def cancel_callback(wait_result, future_done_set):
        """Cancel wait_result if future_done_set has been cancelled."""
        if future_done_set.cancelled() and not wait_result.done():
            wait_result.cancel()

    @coroutine
    def fetch_wait_result(scheduler, first, loop=None):
        if first:
            yield scheduler.async_start()

        # If the current coroutine awakens just after a call to
        # done_callback but before scheduler has been notified of
        # corresponding done future(s), then wait here until scheduler
        # is notified (which will cause future_map to populate).
        while not future_map and scheduler.poll() is None:
            yield asyncio.sleep(0, loop=loop)

        if not future_map:
            if scheduler.poll() is not None:
                coroutine_return((set(), set()))
            else:
                raise AssertionError('expected non-empty future_map')

        wait_result = yield asyncio.wait(list(future_map.values()),
                                         return_when=asyncio.FIRST_COMPLETED,
                                         loop=loop)

        coroutine_return(wait_result)

    first = True
    try:
        while True:
            wait_result = asyncio.ensure_future(fetch_wait_result(scheduler,
                                                                  first,
                                                                  loop=loop),
                                                loop=loop)
            first = False
            future_done_set = loop.create_future()
            future_done_set.add_done_callback(
                functools.partial(cancel_callback, wait_result))
            wait_result.add_done_callback(
                functools.partial(done_callback, future_done_set))
            yield future_done_set
            if not future_map and scheduler.poll() is not None:
                break
    finally:
        # cleanup in case of interruption by SIGINT, etc
        scheduler.cancel()
        scheduler.wait()
Example #5
0
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
	"""
	An asynchronous version of iter_completed. This yields futures, which
	when done, result in a set of input futures that are done. This serves
	as a wrapper around portage's internal TaskScheduler class, using
	standard asyncio interfaces.

	@param futures: iterator of asyncio.Future (or compatible)
	@type futures: iterator
	@param max_jobs: max number of futures to process concurrently (default
		is portage.util.cpuinfo.get_cpu_count())
	@type max_jobs: int
	@param max_load: max load allowed when scheduling a new future,
		otherwise schedule no more than 1 future at a time (default
		is portage.util.cpuinfo.get_cpu_count())
	@type max_load: int or float
	@param loop: event loop
	@type loop: EventLoop
	@return: iterator of futures, which when done, result in a set of
		input futures that are done
	@rtype: iterator
	"""
	loop = asyncio._wrap_loop(loop)

	max_jobs = max_jobs or get_cpu_count()
	max_load = max_load or get_cpu_count()

	future_map = {}
	def task_generator():
		for future in futures:
			future_map[id(future)] = future
			yield AsyncTaskFuture(future=future)

	scheduler = TaskScheduler(
		task_generator(),
		max_jobs=max_jobs,
		max_load=max_load,
		event_loop=loop)

	def done_callback(future_done_set, wait_result):
		"""Propagate results from wait_result to future_done_set."""
		if future_done_set.cancelled():
			return
		done, pending = wait_result.result()
		for future in done:
			del future_map[id(future)]
		future_done_set.set_result(done)

	def cancel_callback(wait_result, future_done_set):
		"""Cancel wait_result if future_done_set has been cancelled."""
		if future_done_set.cancelled() and not wait_result.done():
			wait_result.cancel()

	try:
		scheduler.start()

		# scheduler should ensure that future_map is non-empty until
		# task_generator is exhausted
		while future_map:
			wait_result = asyncio.ensure_future(
				asyncio.wait(list(future_map.values()),
				return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop)
			future_done_set = loop.create_future()
			future_done_set.add_done_callback(
				functools.partial(cancel_callback, wait_result))
			wait_result.add_done_callback(
				functools.partial(done_callback, future_done_set))
			yield future_done_set
	finally:
		# cleanup in case of interruption by SIGINT, etc
		scheduler.cancel()
		scheduler.wait()