def test_call_at(self): loop = get_event_loop() d1 = Future() d2 = Future() c1 = loop.call_at(loop.time()+1, lambda: d1.set_result(loop.time())) c2 = loop.call_later(1, lambda: d2.set_result(loop.time())) t1, t2 = yield pulsar.multi_async((d1, d2)) self.assertTrue(t1 <= t2)
def test_call_at(self): loop = get_event_loop() d1 = Future() d2 = Future() c1 = loop.call_at(loop.time() + 1, lambda: d1.set_result(loop.time())) c2 = loop.call_later(1, lambda: d2.set_result(loop.time())) t1, t2 = yield pulsar.multi_async((d1, d2)) self.assertTrue(t1 <= t2)
async def test_json_with_async_string2(self): d = Future() astr = wsgi.String(d) response = wsgi.Json({'bla': astr}) self.assertEqual(len(response.children), 1) result = response.render() self.assertIsInstance(result, Future) d.set_result('ciao') result = await result self.assertEqual(result, json.dumps({'bla': 'ciao'}))
def test_multi(self): d1 = Future() d2 = Future() d = multi_async([d1, d2, 'bla']) self.assertFalse(d.done()) d2.set_result('first') self.assertFalse(d.done()) d1.set_result('second') result = yield from d self.assertEqual(result, ['second', 'first', 'bla'])
async def test_multi(self): d1 = Future() d2 = Future() d = multi_async([d1, d2, 'bla']) self.assertFalse(d.done()) d2.set_result('first') self.assertFalse(d.done()) d1.set_result('second') result = await d self.assertEqual(result, ['second', 'first', 'bla'])
def test_multi(self): d1 = Future() d2 = Future() d = multi_async([d1, d2, "bla"]) self.assertFalse(d.done()) d2.set_result("first") self.assertFalse(d.done()) d1.set_result("second") result = yield from d self.assertEqual(result, ["second", "first", "bla"])
def test_json_with_async_string2(self): d = Future() astr = wsgi.AsyncString(d) response = wsgi.Json({"bla": astr}) self.assertEqual(len(response.children), 1) result = response.render() self.assertIsInstance(result, Future) d.set_result("ciao") result = yield from result self.assertEqual(result, json.dumps({"bla": "ciao"}))
def test_call_soon(self): ioloop = get_event_loop() tid = yield loop_thread_id(ioloop) d = Future() callback = lambda: d.set_result(current_thread().ident) cbk = ioloop.call_soon(callback) self.assertEqual(cbk._callback, callback) self.assertEqual(cbk._args, ()) # we should be able to wait less than a second result = yield d self.assertEqual(result, tid)
def test_call_later(self): ioloop = get_event_loop() tid = yield loop_thread_id(ioloop) d = Future() timeout1 = ioloop.call_later( 20, lambda: d.set_result(current_thread().ident)) timeout2 = ioloop.call_later( 10, lambda: d.set_result(current_thread().ident)) # lets wake the ioloop self.assertTrue(has_callback(ioloop, timeout1)) self.assertTrue(has_callback(ioloop, timeout2)) timeout1.cancel() timeout2.cancel() self.assertTrue(timeout1._cancelled) self.assertTrue(timeout2._cancelled) timeout1 = ioloop.call_later( 0.1, lambda: d.set_result(current_thread().ident)) yield d self.assertTrue(d.done()) self.assertEqual(d.result(), tid) self.assertFalse(has_callback(ioloop, timeout1))
class GreenPool(AsyncObject): """A pool of running greenlets. This pool maintains a group of greenlets to perform asynchronous tasks via the :meth:`submit` method. """ worker_name = 'exec' def __init__(self, max_workers=None, loop=None): self._loop = loop or get_event_loop() self._max_workers = min(max_workers or _DEFAULT_WORKERS, _MAX_WORKERS) self._greenlets = set() self._available = set() self._queue = deque() self._shutdown = False self._waiter = None self._logger = logging.getLogger('pulsar.greenpool') self._shutdown_lock = threading.Lock() self.wait = wait @property def max_workers(self): return self._max_workers @max_workers.setter def max_workers(self, value): value = int(value) assert value > 0 self._max_workers = value @property def in_green_worker(self): """True if the current greenlet is a green pool worker """ return isinstance(getcurrent(), GreenletWorker) def submit(self, func, *args, **kwargs): """Equivalent to ``func(*args, **kwargs)``. This method create a new task for function ``func`` and adds it to the queue. Return a :class:`~asyncio.Future` called back once the task has finished. """ with self._shutdown_lock: if self._shutdown: raise RuntimeError( 'cannot schedule new futures after shutdown') if self.in_green_worker: return wait(func(*args, **kwargs)) else: future = Future(loop=self._loop) self._put((future, func, args, kwargs)) return future def shutdown(self, wait=True): with self._shutdown_lock: self._shutdown = True self._put(None) if wait: self._waiter = Future(loop=self._loop) return self._waiter def getcurrent(self): return getcurrent() # INTERNALS def _adjust_greenlet_count(self): if (not self._shutdown and not self._available and len(self._greenlets) < self._max_workers): green = GreenletWorker(self._green_run) self._greenlets.add(green) self.logger.debug('Num greenlets: %d', len(self._greenlets)) green.switch() return self._available def _put(self, task): # Run in the main greenlet of the evnet-loop thread self._queue.appendleft(task) self._check_queue() def _check_queue(self): # Run in the main greenlet of the event-loop thread if not self._adjust_greenlet_count(): self.logger.debug('No greenlet available') return self._loop.call_soon(self._check_queue) try: task = self._queue.pop() except IndexError: return ensure_future(self._green_task(self._available.pop(), task), loop=self._loop) async def _green_task(self, green, task): # Coroutine executing the in main greenlet # This coroutine is executed for every task put into the queue while task is not _DONE: # switch to the greenlet to start the task task = green.switch(task) # if an asynchronous result is returned, await while isawaitable(task): try: task = await task except Exception as exc: # This call can return an asynchronous component exc_info = sys.exc_info() if not exc_info[0]: exc_info = (exc, None, None) task = green.throw(*exc_info) def _green_run(self): # The run method of a worker greenlet task = True while task: green = getcurrent() parent = green.parent assert parent # add greenlet in the available greenlets self._available.add(green) task = parent.switch(_DONE) # switch back to the main execution if task: future, func, args, kwargs = task try: try: result = wait(func(*args, **kwargs), True) except StopIteration as exc: # See PEP 479 raise RuntimeError('Unhandled StopIteration') from exc except Exception as exc: future.set_exception(exc) else: future.set_result(result) else: # Greenlet cleanup self._greenlets.remove(green) if self._greenlets: self._put(None) elif self._waiter: self._waiter.set_result(None) self._waiter = None parent.switch(_DONE)
class GreenPool(AsyncObject): '''A pool of running greenlets. This pool maintains a group of greenlets to perform asynchronous tasks via the :meth:`submit` method. ''' worker_name = 'exec' def __init__(self, max_workers=None, loop=None): self._loop = loop or get_event_loop() self._max_workers = min(max_workers or _DEFAULT_WORKERS, _MAX_WORKERS) self._greenlets = set() self._available = set() self._queue = deque() self._shutdown = False self._waiter = None self._logger = logging.getLogger('pulsar.greenpool') self._shutdown_lock = threading.Lock() @property def max_workers(self): return self._max_workers @max_workers.setter def max_workers(self, value): value = int(value) assert value > 0 self._max_workers = value def submit(self, func, *args, **kwargs): '''Equivalent to ``func(*args, **kwargs)``. This method create a new task for function ``func`` and adds it to the queue. Return a :class:`~asyncio.Future` called back once the task has finished. ''' with self._shutdown_lock: if self._shutdown: raise RuntimeError( 'cannot schedule new futures after shutdown') future = Future(loop=self._loop) self._put((future, func, args, kwargs)) return future def shutdown(self, wait=True): with self._shutdown_lock: self._shutdown = True self._put(None) if wait: self._waiter = Future(loop=self._loop) return self._waiter # INTERNALS def _adjust_greenlet_count(self): if (not self._shutdown and not self._available and len(self._greenlets) < self._max_workers): green = GreenletWorker(self._green_run) self._greenlets.add(green) self.logger.debug('Num greenlets: %d', len(self._greenlets)) green.switch() return self._available def _put(self, task): # Run in the main greenlet of the evnet-loop thread self._queue.appendleft(task) self._check_queue() def _check_queue(self): # Run in the main greenlet of the event-loop thread if not self._adjust_greenlet_count(): self.logger.debug('No greenlet available') return self._loop.call_soon(self._check_queue) try: task = self._queue.pop() except IndexError: return async (self._green_task(self._available.pop(), task), loop=self._loop) def _green_task(self, green, task): # Coroutine executing the in main greenlet # This coroutine is executed for every task put into the queue while task is not _DONE: # switch to the greenlet to start the task task = green.switch(task) # if an asynchronous result is returned, yield from while is_async(task): try: task = yield from task except Exception as exc: # This call can return an asynchronous component task = green.throw(exc) def _green_run(self): # The run method of a worker greenlet task = True while task: green = getcurrent() parent = green.parent assert parent # add greenlet in the available greenlets self._available.add(green) task = parent.switch(_DONE) # switch back to the main execution if task: future, func, args, kwargs = task try: result = func(*args, **kwargs) except Exception as exc: future.set_exception(exc) else: future.set_result(result) else: # Greenlet cleanup self._greenlets.remove(green) if self._greenlets: self._put(None) elif self._waiter: self._waiter.set_result(None) self._waiter = None parent.switch(_DONE)
class GreenPool(AsyncObject): '''A pool of running greenlets. This pool maintains a group of greenlets to perform asynchronous tasks via the :meth:`submit` method. ''' worker_name = 'exec' def __init__(self, max_workers=None, loop=None, maxtasks=None): self._loop = loop or get_event_loop() self._max_workers = min(max_workers or _DEFAULT_WORKERS, _MAX_WORKERS) self._greenlets = set() self._available = set() self._maxtasks = maxtasks self._queue = deque() self._shutdown = False self._waiter = None self._shutdown_lock = threading.Lock() def submit(self, func, *args, **kwargs): '''Equivalent to ``func(*args, **kwargs)``. This method create a new task for function ``func`` and adds it to the queue. Return a :class:`~asyncio.Future` called back once the task has finished. ''' with self._shutdown_lock: if self._shutdown: raise RuntimeError( 'cannot schedule new futures after shutdown') future = Future(loop=self._loop) self._put((future, func, args, kwargs)) return future def shutdown(self, wait=True): with self._shutdown_lock: self._shutdown = True self._put() if wait: self._waiter = Future(loop=self._loop) return self._waiter # INTERNALS def _adjust_greenlet_count(self): if not self._available and len(self._greenlets) < self._max_workers: greenlet = GreenletWorker(self._green_run) self._greenlets.add(greenlet) greenlet.switch() def _put(self, task=None): # Run in the main greenlet of the evnet-loop thread if task: self._adjust_greenlet_count() self._queue.appendleft(task) self._check_queue() def _check_queue(self): # Run in the main greenlet of the event-loop thread if not self._available: return try: task = self._queue.pop() except IndexError: return async(self._green_task(self._available.pop(), task), loop=self._loop) def _green_task(self, greenlet, task): # Run in the main greenlet of the event-loop thread while task is not _DONE: # switch to the greenlet to start the task task = greenlet.switch(task) # if an asynchronous result is returned, yield from while is_async(task): try: task = yield from task except Exception as exc: # This call can return an asynchronous component task = greenlet.throw(exc) def _green_run(self): # The run method of a worker greenlet task = True while task: greenlet = getcurrent() parent = greenlet.parent assert parent self._available.add(greenlet) self._loop.call_soon(self._check_queue) task = parent.switch(_DONE) # switch back to the main execution if task: # If a new task is available execute it # Here we are in the child greenlet future, func, args, kwargs = task try: result = func(*args, **kwargs) except Exception as exc: future.set_exception(exc) else: future.set_result(result) else: # Greenlet cleanup self._greenlets.remove(greenlet) if self._greenlets: self._put(None) elif self._waiter: self._waiter.set_result(None) self._waiter = None parent.switch(_DONE)