def testAsyncCancel(self):

		loop = global_event_loop()
		input_futures = set()
		future_count = 3

		def future_generator():
			for i in range(future_count):
				future = loop.create_future()
				loop.call_soon(lambda future: None if future.done()
					else future.set_result(None), future)
				input_futures.add(future)
				yield future

		for future_done_set in async_iter_completed(future_generator(),
			max_jobs=True, max_load=True, loop=loop):
			future_done_set.cancel()
			break

		# With max_jobs=True, async_iter_completed should have executed
		# the generator until it raised StopIteration.
		self.assertEqual(future_count, len(input_futures))

		loop.run_until_complete(asyncio.wait(input_futures, loop=loop))

		# The futures may have results or they may have been cancelled
		# by TaskScheduler, and behavior varies depending on the python
		# interpreter.
		for future in input_futures:
			future.cancelled() or future.result()
示例#2
0
	def _testPipeLoggerToPipe(self, test_string, loop=None):
		"""
		Test PipeLogger writing to a pipe connected to a PipeReader.
		This verifies that PipeLogger does not deadlock when writing
		to a pipe that's drained by a PipeReader running in the same
		process (requires non-blocking write).
		"""

		input_fd, writer_pipe = os.pipe()
		_set_nonblocking(writer_pipe)
		writer_pipe = os.fdopen(writer_pipe, 'wb', 0)
		writer = asyncio.ensure_future(_writer(writer_pipe, test_string.encode('ascii'), loop=loop), loop=loop)
		writer.add_done_callback(lambda writer: writer_pipe.close())

		pr, pw = os.pipe()

		consumer = PipeLogger(background=True,
			input_fd=input_fd,
			log_file_path=os.fdopen(pw, 'wb', 0),
			scheduler=loop)
		consumer.start()

		# Before starting the reader, wait here for a moment, in order
		# to exercise PipeLogger's handling of EAGAIN during write.
		yield asyncio.wait([writer], timeout=0.01)

		reader = _reader(pr, loop=loop)
		yield writer
		content = yield reader
		yield consumer.async_wait()

		self.assertEqual(consumer.returncode, os.EX_OK)

		coroutine_return(content.decode('ascii', 'replace'))
示例#3
0
	def testHangForeverReraise(self):
		loop = global_event_loop()
		func_coroutine = self._wrap_coroutine_func(HangForever())
		decorator = retry(reraise=True, try_max=2, try_timeout=0.1,
			delay_func=RandomExponentialBackoff(multiplier=0.1, base=2))
		decorated_func = decorator(func_coroutine, loop=loop)
		done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop))
		self.assertEqual(len(done), 1)
		self.assertTrue(isinstance(done.pop().exception(), asyncio.TimeoutError))
示例#4
0
	def testSucceedNever(self):
		loop = global_event_loop()
		func_coroutine = self._wrap_coroutine_func(SucceedNever())
		decorator = retry(try_max=4, try_timeout=None,
			delay_func=RandomExponentialBackoff(multiplier=0.1, base=2))
		decorated_func = decorator(func_coroutine, loop=loop)
		done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop))
		self.assertEqual(len(done), 1)
		self.assertTrue(isinstance(done.pop().exception().__cause__, SucceedNeverException))
示例#5
0
	def testOverallTimeoutWithTimeoutError(self):
		loop = global_event_loop()
		# results in TimeoutError because it hangs forever
		func_coroutine = self._wrap_coroutine_func(HangForever())
		decorator = retry(try_timeout=0.1, overall_timeout=0.3,
			delay_func=RandomExponentialBackoff(multiplier=0.1, base=2))
		decorated_func = decorator(func_coroutine, loop=loop)
		done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop))
		self.assertEqual(len(done), 1)
		self.assertTrue(isinstance(done.pop().exception().__cause__, asyncio.TimeoutError))
示例#6
0
	def testCancelRetry(self):
		loop = global_event_loop()
		func_coroutine = self._wrap_coroutine_func(SucceedNever())
		decorator = retry(try_timeout=0.1,
			delay_func=RandomExponentialBackoff(multiplier=0.1, base=2))
		decorated_func = decorator(func_coroutine, loop=loop)
		future = decorated_func()
		loop.call_later(0.3, future.cancel)
		done, pending = loop.run_until_complete(asyncio.wait([future], loop=loop))
		self.assertEqual(len(done), 1)
		self.assertTrue(done.pop().cancelled())
示例#7
0
 def testCancelRetry(self):
     loop = global_event_loop()
     func_coroutine = self._wrap_coroutine_func(SucceedNever())
     decorator = retry(try_timeout=0.1,
                       delay_func=RandomExponentialBackoff(multiplier=0.1,
                                                           base=2))
     decorated_func = decorator(func_coroutine, loop=loop)
     future = decorated_func()
     loop.call_later(0.3, future.cancel)
     done, pending = loop.run_until_complete(
         asyncio.wait([future], loop=loop))
     self.assertEqual(len(done), 1)
     self.assertTrue(done.pop().cancelled())
示例#8
0
 def testHangForever(self):
     loop = global_event_loop()
     func_coroutine = self._wrap_coroutine_func(HangForever())
     decorator = retry(try_max=2,
                       try_timeout=0.1,
                       delay_func=RandomExponentialBackoff(multiplier=0.1,
                                                           base=2))
     decorated_func = decorator(func_coroutine, loop=loop)
     done, pending = loop.run_until_complete(
         asyncio.wait([decorated_func()], loop=loop))
     self.assertEqual(len(done), 1)
     self.assertTrue(
         isinstance(done.pop().exception().__cause__, asyncio.TimeoutError))
示例#9
0
	def _wait_loop(self, timeout=None):
		loop = self.scheduler
		tasks = [self.async_wait()]
		if timeout is not None:
			tasks.append(asyncio.ensure_future(
				asyncio.sleep(timeout, loop=loop), loop=loop))
		try:
			loop.run_until_complete(asyncio.ensure_future(
				asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED,
				loop=loop), loop=loop))
		finally:
			for task in tasks:
				task.cancel()
示例#10
0
	def _wait_loop(self, timeout=None):
		loop = self.scheduler
		tasks = [self.async_wait()]
		if timeout is not None:
			tasks.append(asyncio.ensure_future(
				asyncio.sleep(timeout, loop=loop), loop=loop))
		try:
			loop.run_until_complete(asyncio.ensure_future(
				asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED,
				loop=loop), loop=loop))
		finally:
			for task in tasks:
				task.cancel()
示例#11
0
 def testOverallTimeoutWithException(self):
     loop = global_event_loop()
     func_coroutine = self._wrap_coroutine_func(SucceedNever())
     decorator = retry(try_timeout=0.1,
                       overall_timeout=0.3,
                       delay_func=RandomExponentialBackoff(multiplier=0.1,
                                                           base=2))
     decorated_func = decorator(func_coroutine, loop=loop)
     done, pending = loop.run_until_complete(
         asyncio.wait([decorated_func()], loop=loop))
     self.assertEqual(len(done), 1)
     self.assertTrue(
         isinstance(done.pop().exception().__cause__,
                    SucceedNeverException))
示例#12
0
 def testSucceedNeverReraise(self):
     loop = global_event_loop()
     with self._wrap_coroutine_func(SucceedNever()) as func_coroutine:
         decorator = retry(reraise=True,
                           try_max=4,
                           try_timeout=None,
                           delay_func=RandomExponentialBackoff(
                               multiplier=0.1, base=2))
         decorated_func = decorator(func_coroutine, loop=loop)
         done, pending = loop.run_until_complete(
             asyncio.wait([decorated_func()], loop=loop))
         self.assertEqual(len(done), 1)
         self.assertTrue(
             isinstance(done.pop().exception(), SucceedNeverException))
示例#13
0
 def testSucceedNever(self):
     loop = global_event_loop()._asyncio_wrapper
     func = SucceedNever()
     func_coroutine = functools.partial(loop.run_in_executor, None, func)
     decorator = retry(try_max=4,
                       try_timeout=None,
                       delay_func=RandomExponentialBackoff(multiplier=0.1,
                                                           base=2))
     decorated_func = decorator(func_coroutine)
     done, pending = loop.run_until_complete(
         asyncio.wait([decorated_func()], loop=loop))
     self.assertEqual(len(done), 1)
     self.assertTrue(
         isinstance(done.pop().exception().__cause__,
                    SucceedNeverException))
示例#14
0
 def testOverallTimeoutWithTimeoutError(self):
     loop = global_event_loop()._asyncio_wrapper
     # results in TimeoutError because it hangs forever
     func = HangForever()
     func_coroutine = functools.partial(loop.run_in_executor, None, func)
     decorator = retry(try_timeout=0.1,
                       overall_timeout=0.3,
                       delay_func=RandomExponentialBackoff(multiplier=0.1,
                                                           base=2))
     decorated_func = decorator(func_coroutine)
     done, pending = loop.run_until_complete(
         asyncio.wait([decorated_func()], loop=loop))
     self.assertEqual(len(done), 1)
     self.assertTrue(
         isinstance(done.pop().exception().__cause__, asyncio.TimeoutError))
示例#15
0
 def testHangForeverReraise(self):
     loop = global_event_loop()._asyncio_wrapper
     func = HangForever()
     func_coroutine = functools.partial(loop.run_in_executor, None, func)
     decorator = retry(reraise=True,
                       try_max=2,
                       try_timeout=0.1,
                       delay_func=RandomExponentialBackoff(multiplier=0.1,
                                                           base=2))
     decorated_func = decorator(func_coroutine)
     done, pending = loop.run_until_complete(
         asyncio.wait([decorated_func()], loop=loop))
     self.assertEqual(len(done), 1)
     self.assertTrue(
         isinstance(done.pop().exception(), asyncio.TimeoutError))
示例#16
0
    def test_cancelled_future(self):
        """
		When a coroutine raises CancelledError, the coroutine's
		future is cancelled.
		"""
        @coroutine
        def cancelled_future_coroutine(loop=None):
            loop = asyncio._wrap_loop(loop)
            while True:
                future = loop.create_future()
                loop.call_soon(future.cancel)
                yield future

        loop = asyncio.get_event_loop()
        future = loop.run_until_complete(
            asyncio.wait([cancelled_future_coroutine()]))[0].pop()
        self.assertTrue(future.cancelled())
示例#17
0
	def test_cancelled_future(self):
		"""
		When a coroutine raises CancelledError, the coroutine's
		future is cancelled.
		"""

		@coroutine
		def cancelled_future_coroutine(loop=None):
			loop = asyncio._wrap_loop(loop)
			while True:
				future = loop.create_future()
				loop.call_soon(future.cancel)
				yield future

		loop = asyncio.get_event_loop()
		future = loop.run_until_complete(asyncio.wait([cancelled_future_coroutine()]))[0].pop()
		self.assertTrue(future.cancelled())
示例#18
0
    def fetch_wait_result(scheduler, first, loop=None):
        if first:
            yield scheduler.async_start()

        # If the current coroutine awakens just after a call to
        # done_callback but before scheduler has been notified of
        # corresponding done future(s), then wait here until scheduler
        # is notified (which will cause future_map to populate).
        while not future_map and scheduler.poll() is None:
            yield asyncio.sleep(0, loop=loop)

        if not future_map:
            if scheduler.poll() is not None:
                coroutine_return((set(), set()))
            else:
                raise AssertionError('expected non-empty future_map')

        wait_result = yield asyncio.wait(list(future_map.values()),
                                         return_when=asyncio.FIRST_COMPLETED,
                                         loop=loop)

        coroutine_return(wait_result)
示例#19
0
    def test_method_coroutine(self):
        class Cubby:

            _empty = object()

            def __init__(self, loop):
                self._loop = loop
                self._value = self._empty
                self._waiters = []

            def _notify(self):
                waiters = self._waiters
                self._waiters = []
                for waiter in waiters:
                    waiter.cancelled() or waiter.set_result(None)

            def _wait(self):
                waiter = self._loop.create_future()
                self._waiters.append(waiter)
                return waiter

            @coroutine
            def read(self, loop=None):
                while self._value is self._empty:
                    yield self._wait()

                value = self._value
                self._value = self._empty
                self._notify()
                coroutine_return(value)

            @coroutine
            def write(self, value, loop=None):
                while self._value is not self._empty:
                    yield self._wait()

                self._value = value
                self._notify()

        @coroutine
        def writer_coroutine(cubby, values, sentinel, loop=None):
            for value in values:
                yield cubby.write(value, loop=loop)
            yield cubby.write(sentinel, loop=loop)

        @coroutine
        def reader_coroutine(cubby, sentinel, loop=None):
            results = []
            while True:
                result = yield cubby.read(loop=loop)
                if result == sentinel:
                    break
                results.append(result)
            coroutine_return(results)

        loop = asyncio.get_event_loop()
        cubby = Cubby(loop)
        values = list(range(3))
        writer = asyncio.ensure_future(writer_coroutine(cubby,
                                                        values,
                                                        None,
                                                        loop=loop),
                                       loop=loop)
        reader = asyncio.ensure_future(reader_coroutine(cubby, None,
                                                        loop=loop),
                                       loop=loop)
        loop.run_until_complete(asyncio.wait([writer, reader], loop=loop))

        self.assertEqual(reader.result(), values)

        # Test decoration of coroutine methods and functions for
        # synchronous usage, allowing coroutines to smoothly
        # blend with synchronous code.
        sync_cubby = _sync_methods(cubby, loop=loop)
        sync_reader = _sync_decorator(reader_coroutine, loop=loop)
        writer = asyncio.ensure_future(writer_coroutine(cubby,
                                                        values,
                                                        None,
                                                        loop=loop),
                                       loop=loop)
        self.assertEqual(sync_reader(cubby, None), values)
        self.assertTrue(writer.done())

        for i in range(3):
            sync_cubby.write(i)
            self.assertEqual(sync_cubby.read(), i)
示例#20
0
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
	"""
	An asynchronous version of iter_completed. This yields futures, which
	when done, result in a set of input futures that are done. This serves
	as a wrapper around portage's internal TaskScheduler class, using
	standard asyncio interfaces.

	@param futures: iterator of asyncio.Future (or compatible)
	@type futures: iterator
	@param max_jobs: max number of futures to process concurrently (default
		is portage.util.cpuinfo.get_cpu_count())
	@type max_jobs: int
	@param max_load: max load allowed when scheduling a new future,
		otherwise schedule no more than 1 future at a time (default
		is portage.util.cpuinfo.get_cpu_count())
	@type max_load: int or float
	@param loop: event loop
	@type loop: EventLoop
	@return: iterator of futures, which when done, result in a set of
		input futures that are done
	@rtype: iterator
	"""
	loop = asyncio._wrap_loop(loop)

	max_jobs = max_jobs or get_cpu_count()
	max_load = max_load or get_cpu_count()

	future_map = {}
	def task_generator():
		for future in futures:
			future_map[id(future)] = future
			yield AsyncTaskFuture(future=future)

	scheduler = TaskScheduler(
		task_generator(),
		max_jobs=max_jobs,
		max_load=max_load,
		event_loop=loop)

	def done_callback(future_done_set, wait_result):
		"""Propagate results from wait_result to future_done_set."""
		if future_done_set.cancelled():
			return
		done, pending = wait_result.result()
		for future in done:
			del future_map[id(future)]
		future_done_set.set_result(done)

	def cancel_callback(wait_result, future_done_set):
		"""Cancel wait_result if future_done_set has been cancelled."""
		if future_done_set.cancelled() and not wait_result.done():
			wait_result.cancel()

	try:
		scheduler.start()

		# scheduler should ensure that future_map is non-empty until
		# task_generator is exhausted
		while future_map:
			wait_result = asyncio.ensure_future(
				asyncio.wait(list(future_map.values()),
				return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop)
			future_done_set = loop.create_future()
			future_done_set.add_done_callback(
				functools.partial(cancel_callback, wait_result))
			wait_result.add_done_callback(
				functools.partial(done_callback, future_done_set))
			yield future_done_set
	finally:
		# cleanup in case of interruption by SIGINT, etc
		scheduler.cancel()
		scheduler.wait()
示例#21
0
	def test_method_coroutine(self):

		class Cubby(object):

			_empty = object()

			def __init__(self, loop):
				self._loop = loop
				self._value = self._empty
				self._waiters = []

			def _notify(self):
				waiters = self._waiters
				self._waiters = []
				for waiter in waiters:
					waiter.cancelled() or waiter.set_result(None)

			def _wait(self):
				waiter = self._loop.create_future()
				self._waiters.append(waiter)
				return waiter

			@coroutine
			def read(self):
				while self._value is self._empty:
					yield self._wait()

				value = self._value
				self._value = self._empty
				self._notify()
				coroutine_return(value)

			@coroutine
			def write(self, value):
				while self._value is not self._empty:
					yield self._wait()

				self._value = value
				self._notify()

		@coroutine
		def writer_coroutine(cubby, values, sentinel):
			for value in values:
				yield cubby.write(value)
			yield cubby.write(sentinel)

		@coroutine
		def reader_coroutine(cubby, sentinel):
			results = []
			while True:
				result = yield cubby.read()
				if result == sentinel:
					break
				results.append(result)
			coroutine_return(results)

		loop = asyncio.get_event_loop()
		cubby = Cubby(loop)
		values = list(range(3))
		writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop)
		reader = asyncio.ensure_future(reader_coroutine(cubby, None), loop=loop)
		loop.run_until_complete(asyncio.wait([writer, reader]))

		self.assertEqual(reader.result(), values)
示例#22
0
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
    """
	An asynchronous version of iter_completed. This yields futures, which
	when done, result in a set of input futures that are done. This serves
	as a wrapper around portage's internal TaskScheduler class, using
	standard asyncio interfaces.

	@param futures: iterator of asyncio.Future (or compatible)
	@type futures: iterator
	@param max_jobs: max number of futures to process concurrently (default
		is portage.util.cpuinfo.get_cpu_count())
	@type max_jobs: int
	@param max_load: max load allowed when scheduling a new future,
		otherwise schedule no more than 1 future at a time (default
		is portage.util.cpuinfo.get_cpu_count())
	@type max_load: int or float
	@param loop: event loop
	@type loop: EventLoop
	@return: iterator of futures, which when done, result in a set of
		input futures that are done
	@rtype: iterator
	"""
    loop = asyncio._wrap_loop(loop)

    max_jobs = max_jobs or get_cpu_count()
    max_load = max_load or get_cpu_count()

    future_map = {}

    def task_generator():
        for future in futures:
            future_map[id(future)] = future
            yield AsyncTaskFuture(future=future)

    scheduler = TaskScheduler(task_generator(),
                              max_jobs=max_jobs,
                              max_load=max_load,
                              event_loop=loop)

    def done_callback(future_done_set, wait_result):
        """Propagate results from wait_result to future_done_set."""
        if future_done_set.cancelled():
            return
        done, pending = wait_result.result()
        for future in done:
            del future_map[id(future)]
        future_done_set.set_result(done)

    def cancel_callback(wait_result, future_done_set):
        """Cancel wait_result if future_done_set has been cancelled."""
        if future_done_set.cancelled() and not wait_result.done():
            wait_result.cancel()

    try:
        scheduler.start()

        # scheduler should ensure that future_map is non-empty until
        # task_generator is exhausted
        while future_map:
            wait_result = asyncio.ensure_future(asyncio.wait(
                list(future_map.values()),
                return_when=asyncio.FIRST_COMPLETED,
                loop=loop),
                                                loop=loop)
            future_done_set = loop.create_future()
            future_done_set.add_done_callback(
                functools.partial(cancel_callback, wait_result))
            wait_result.add_done_callback(
                functools.partial(done_callback, future_done_set))
            yield future_done_set
    finally:
        # cleanup in case of interruption by SIGINT, etc
        scheduler.cancel()
        scheduler.wait()
示例#23
0
	def test_method_coroutine(self):

		class Cubby(object):

			_empty = object()

			def __init__(self, loop):
				self._loop = loop
				self._value = self._empty
				self._waiters = []

			def _notify(self):
				waiters = self._waiters
				self._waiters = []
				for waiter in waiters:
					waiter.cancelled() or waiter.set_result(None)

			def _wait(self):
				waiter = self._loop.create_future()
				self._waiters.append(waiter)
				return waiter

			@coroutine
			def read(self):
				while self._value is self._empty:
					yield self._wait()

				value = self._value
				self._value = self._empty
				self._notify()
				coroutine_return(value)

			@coroutine
			def write(self, value):
				while self._value is not self._empty:
					yield self._wait()

				self._value = value
				self._notify()

		@coroutine
		def writer_coroutine(cubby, values, sentinel):
			for value in values:
				yield cubby.write(value)
			yield cubby.write(sentinel)

		@coroutine
		def reader_coroutine(cubby, sentinel):
			results = []
			while True:
				result = yield cubby.read()
				if result == sentinel:
					break
				results.append(result)
			coroutine_return(results)

		loop = asyncio.get_event_loop()
		cubby = Cubby(loop)
		values = list(range(3))
		writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop)
		reader = asyncio.ensure_future(reader_coroutine(cubby, None), loop=loop)
		loop.run_until_complete(asyncio.wait([writer, reader]))

		self.assertEqual(reader.result(), values)

		# Test decoration of coroutine methods and functions for
		# synchronous usage, allowing coroutines to smoothly
		# blend with synchronous code.
		sync_cubby = _sync_methods(cubby, loop=loop)
		sync_reader = _sync_decorator(reader_coroutine, loop=loop)
		writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop)
		self.assertEqual(sync_reader(cubby, None), values)
		self.assertTrue(writer.done())

		for i in range(3):
			sync_cubby.write(i)
			self.assertEqual(sync_cubby.read(), i)