def test_is_coroutine_function(self): self.finished = True def f(): yield gen.moment coro = gen.coroutine(f) self.assertFalse(gen.is_coroutine_function(f)) self.assertTrue(gen.is_coroutine_function(coro)) self.assertFalse(gen.is_coroutine_function(coro()))
def test_is_coroutine_function(self): self.finished = True def f(): yield gen.moment coro = gen.coroutine(f) self.assertFalse(gen.is_coroutine_function(f)) self.assertTrue(gen.is_coroutine_function(coro)) self.assertFalse(gen.is_coroutine_function(coro()))
def test_stop_after_attempt(self): assert gen.is_coroutine_function(_retryable_coroutine) thing = NoIOErrorAfterCount(2) try: yield _retryable_coroutine_with_2_attempts(thing) except RetryError: assert thing.counter == 2
def add_action(self, action: Union[Callable, Coroutine], interval: float, initial_delay: float = None) -> None: """ Add a new action :param action: A function to call periodically :param interval: The interval between execution of actions :param initial_delay: Delay to the first execution, defaults to interval """ assert inspect.iscoroutinefunction(action) or gen.is_coroutine_function(action) if self._stopped: LOGGER.warning("Scheduling action '%s', while scheduler is stopped", action.__name__) return if initial_delay is None: initial_delay = interval LOGGER.debug("Scheduling action %s every %d seconds with initial delay %d", action, interval, initial_delay) def action_function(): LOGGER.info("Calling %s" % action) if action in self._scheduled: try: ensure_future_and_handle_exception(LOGGER, "Uncaught exception while executing scheduled action", action()) except Exception: LOGGER.exception("Uncaught exception while executing scheduled action") finally: # next iteration ihandle = IOLoop.current().call_later(interval, action_function) self._scheduled[action] = ihandle handle = IOLoop.current().call_later(initial_delay, action_function) self._scheduled[action] = handle
def process_tasks(self): """ Main task processing method (callback to register in ioloop). Consume and process tasks in queue and reschedule processed tasks when relevant. A task is processed if associated data was not removed from scheduler. A task is rescheduled if processing callback returns False (True means `task definitively done`) AND if task deadline is not null. """ while True: task = yield self.tasks_queue.get() # type: _Task try: if task.valid and (not isinstance(task, _ImmediateTask) or task.can_still_process()): if gen.is_coroutine_function(self.callback_process): remove_data = yield self.callback_process(task.data) else: remove_data = self.callback_process(task.data) remove_data = remove_data or not task.deadline.delay with (yield self.lock.acquire()): del self.data_in_queue[task.data] if not remove_data: self.data_in_heap[task.data] = _Deadline( self.current_time, task.deadline.delay) finally: self.tasks_queue.task_done()
def wrapped_callable(self, _callable, *args, **kw_args): """ wrap any kind of callable into chained callable, which will call itself after certain interval """ if is_coroutine_function(_callable): @coroutine def wrapper(): try: yield _callable(*args, **kw_args) except Exception as e: app_log.exception(r'{}'.format(e)) if not self._stop: yield sleep(self.interval) loop = IOLoop.current() loop.call_later(0, wrapper) return wrapper else: def wrapper(): try: _callable(*args, **kw_args) except Exception as e: app_log.exception(r'{}'.format(e)) if not self._stop: loop = IOLoop.current() loop.call_later(self.interval, wrapper) return wrapper
def handle_request(server, request, connection_handler): """ (coroutine) Find request handler function for associated request, run it and return its result. :param server: a Server object to pass to handler function. :param request: a request object to pass to handler function. See diplomacy.communication.requests for possible requests. :param connection_handler: a ConnectionHandler object to pass to handler function. :return: (future) either None or a response object. See module diplomacy.communication.responses for possible responses. """ request_handler_fn = MAPPING.get(type(request), None) if not request_handler_fn: raise exceptions.RequestException() game = server.get_game(request.game_id) # Game not found if not game or game.is_game_completed or game.is_game_canceled: future = Future() future.set_result([responses.REJ(bytes(request))]) return future if gen.is_coroutine_function(request_handler_fn): # Throw the future returned by this coroutine. return request_handler_fn(server, request, connection_handler, game) # Create and return a future. future = Future() try: result = request_handler_fn(server, request, connection_handler, game) future.set_result(result) except exceptions.DiplomacyException as exc: future.set_exception(exc) return future
def run_async(func, *args, **kwargs): if not inspect.iscoroutinefunction(func) and not gen.is_coroutine_function( func): fu = registry.executor_pool().submit(func, *args, **kwargs) fu = to_tornado_future(fu) else: fu = func(*args, **kwargs) return gen.convert_yielded(fu)
def has_keyword(func, keyword): if PY3: return keyword in inspect.signature(func).parameters else: # https://stackoverflow.com/questions/50100498/determine-keywords-of-a-tornado-coroutine if gen.is_coroutine_function(func): func = func.__wrapped__ return keyword in inspect.getargspec(func).args
def has_keyword(func, keyword): if PY3: return keyword in inspect.signature(func).parameters else: # https://stackoverflow.com/questions/50100498/determine-keywords-of-a-tornado-coroutine if gen.is_coroutine_function(func): func = func.__wrapped__ return keyword in inspect.getargspec(func).args
async def wrapper(self, *args, **kwargs): spec = getfullargspec(method) filling_args = spec.args[len(args) + 1:] # 切出需要填充的参数 default_values = spec.defaults[-len(filling_args):] if spec.defaults else [] # 切出需要的默认值 # 倒序,参数与默认值对齐 for key, default in zip_longest(reversed(filling_args), reversed(default_values)): if key in kwargs: continue if key in self._json_args: kwargs[key] = self._json_args.get(key) continue if isinstance(default, list): value = self.get_arguments(key, True) or default else: value = self.get_argument(key, default) kwargs[key] = value # 根据注解做类型转换 model_dict = {} for key, value in kwargs.items(): if key not in spec.annotations: continue annotations = spec.annotations.get(key) try: if issubclass(annotations, BaseModel): model = annotations(context=HandlerContext(self)) kwargs[key] = model model_dict[key] = model elif isinstance(value, list): kwargs[key] = [annotations(item) for item in value] elif value: kwargs[key] = annotations(value) except Exception as e: logging.exception(e) logging.info(f'{key} 字段期待类型为: {str(annotations)} 实际为: "{value}"') self.finish({'code': -1, 'msg': '参数错误'}) return try: # 捕获异常,关闭连接 if iscoroutinefunction(method) or is_coroutine_function(method): response = await method(self, *args, **kwargs) else: response = method(self, *args, **kwargs) return response except ParametersError as e: logging.exception(e) self.finish({'code': -1, 'msg': '参数错误'}) except (NotFound, FileTypeError, Duplicate, PermissionDenied, InternalError) as e: self.finish({'code': e.code, 'msg': e.msg}) except Finish as e: raise e except Exception as e: logging.exception(e) self.finish({'code': -1, 'msg': '内部错误'}) finally: for key, model in model_dict.items(): model.clear()
def wrapper(*args, **kwargs): func_result = yield threadpool_executor.submit(func, *args, **kwargs) if is_coroutine_function(func): result = yield func_result else: result = func_result return result
def addCleanup(self, function, *args, **kwargs): """ Add a function, with arguments, to be called when the test is completed. If function is a coroutine function, it will run on the loop before it's cleaned. """ if gen.is_coroutine_function(function): return self._async_cleanups.append((function, args, kwargs)) return super(BaseTestCase, self).addCleanup(function, *args, **kwargs)
def delete(self, path): path = path.strip('/') self.log.info('Appmode deleting: %s', path) # delete session, including the kernel sm = self.session_manager if gen.is_coroutine_function(sm.get_session): s = yield sm.get_session(path=path) else: s = sm.get_session(path=path) if gen.is_coroutine_function(sm.delete_session): yield sm.delete_session(session_id=s['id']) else: sm.delete_session(session_id=s['id']) # delete tmp copy cm = self.contents_manager cm.delete(path) self.finish()
def ensure_coroutine(func_or_coro): if gen.is_coroutine_function(func_or_coro): return func_or_coro else: @coroutine def coro(*args, **kwargs): raise Return(func_or_coro(*args, **kwargs)) return coro
def post(self, path): assert self.get_body_arguments("appmode_action")[0] == "delete" path = path.strip('/') self.log.info('Appmode deleting: %s', path) # delete session, including the kernel sm = self.session_manager if gen.is_coroutine_function(sm.get_session): s = yield sm.get_session(path=path) else: s = sm.get_session(path=path) if gen.is_coroutine_function(sm.delete_session): yield sm.delete_session(session_id=s['id']) else: sm.delete_session(session_id=s['id']) # delete tmp copy cm = self.contents_manager cm.delete(path) self.finish()
def add_action( self, action: TaskMethod, interval: float, initial_delay: Optional[float] = None, ) -> None: """ Add a new action :param action: A function to call periodically :param interval: The interval between execution of actions :param initial_delay: Delay to the first execution, defaults to interval """ assert inspect.iscoroutinefunction( action) or gen.is_coroutine_function(action) if self._stopped: LOGGER.warning( "Scheduling action '%s', while scheduler is stopped", action.__name__) return if initial_delay is None: initial_delay = interval LOGGER.debug( "Scheduling action %s every %d seconds with initial delay %d", action, interval, initial_delay) def action_function() -> None: LOGGER.info("Calling %s" % action) if action in self._scheduled: try: task = ensure_future_and_handle_exception( logger=LOGGER, msg= "Uncaught exception while executing scheduled action", action=action(), notify_done_callback=functools.partial( self._notify_done, action), ) self._add_to_executing_tasks(action, task) except Exception: LOGGER.exception( "Uncaught exception while executing scheduled action") finally: # next iteration ihandle = IOLoop.current().call_later( interval, action_function) self._scheduled[action] = ihandle handle = IOLoop.current().call_later(initial_delay, action_function) self._scheduled[action] = handle
def start_io_loop(on_start_callback, custom_io_loop=None, stop_on_interrupt=True, **kwargs): """ Starts an asynchronous IO Loop :param on_start_callback: The callback to call when the loop is started. :param custom_io_loop: A custom IO loop object to start, otherwise the .instance() loop will be started. :param stop_on_interrupt: Boolean. If true, the io loop will stop on CTRL-C. :return: Nothing """ if not is_coroutine_function(on_start_callback): LOGGER.error('The on_start_callback must be a gen.coroutine()') raise RuntimeError() io_loop = custom_io_loop or ioloop.IOLoop.instance() io_loop.spawn_callback(on_start_callback, **kwargs) try: io_loop.start() except KeyboardInterrupt: if stop_on_interrupt: io_loop.stop()
def iscoroutinepartial(coro): """ Function returns True if function it's a partial instance of coroutine. See additional information here_. :param coro: Function :return: bool .. _here: https://goo.gl/C0S4sQ """ while True: parent = coro coro = getattr(parent, 'func', None) if coro is None: break return gen.is_coroutine_function(parent)
def submit_task(self, function, *args, **kwargs): async_result = AsyncResult() if self._shutting_down or self._shuted_down: async_result.set_exception( ShutedDownError(self._coroutine_pool_name)) return async_result if not gen.is_coroutine_function(function): async_result.set_exception( RuntimeError("function must be tornado coroutine function")) return async_result is_full = False task_item = TaskItem(function, args, kwargs, async_result) try: self._queue.put_nowait(task_item) async_result.set_time_info("submitted_to_queue_at") except QueueFull: is_full = True if is_full: return self._reject_handler(self._queue, task_item) else: self._core_coroutines_wait_condition.notify() return async_result
def read_stream(self): """ Read the next message from the stream """ messages = [] in_message = yield DaideMessage.from_stream(self.stream) if in_message and in_message.is_valid: message_handler = self.message_mapping.get(in_message.message_type, None) if not message_handler: raise RuntimeError( 'Unrecognized DAIDE message type [{}]'.format( in_message.message_type)) if gen.is_coroutine_function(message_handler): messages = yield message_handler(in_message) else: messages = message_handler(in_message) elif in_message: err_message = ErrorMessage() err_message.error_code = in_message.error_code messages = [err_message] for message in messages: yield self.write_message(message)
def is_coroutine_function(f): return asyncio.iscoroutinefunction(f) or gen.is_coroutine_function(f)
def iscoroutinefunction(f): return inspect.iscoroutinefunction(f) or gen.is_coroutine_function(f)
def test_retry(self): assert gen.is_coroutine_function(_retryable_coroutine) thing = NoIOErrorAfterCount(5) yield _retryable_coroutine(thing) assert thing.counter == thing.count
def _future_stream_connect(self, host=None, port=None, af=socket.AF_UNSPEC, ssl_options=None, max_buffer_size=None, source_ip=None, source_port=None, timeout=None): assert self._future_thread_id == thread.get_ident( ), 'this should run in IO thread:%s' % self._future_thread_id if self.future_connected(): if self._future_periodic_connect.is_running(): self._future_periodic_connect.stop() raise Return(self) if self._future_connecting is True: return else: self._future_connecting = True host = host or self.host port = port or self.port LOG.warn('Connecting %s:%s, threadId:%s, id:%s' % (host, port, thread.get_ident(), id(self))) try: self.future_stream = yield self._future_connector.connect( host, port, timeout=timeout, max_buffer_size=max_buffer_size, af=af, ssl_options=ssl_options, source_ip=source_ip, source_port=source_port) # Note: we first register Async Close|Read|Write callback here, since handshake may require Async READ|WRITE except Exception as e: self._future_connecting = False LOG.exception('Error connect to %s:%s, retry.', host, port) if self.future_stream: self.future_stream.close() if not self._future_periodic_connect.is_running(): self._future_periodic_connect.start() return self._future_callback_queue.clear() # registry self._future_packet_bulk = [] self._future_connecting = False self._future_chunk_size = stream_options(self.future_stream, blocking=False, keepalive=True) self._future_reader_buffer = bytearray(self._future_chunk_size) self.future_stream.set_close_callback(self._future_stream_close) self.future_stream.read_into(self._future_reader_buffer, callback=self._future_socket_read, partial=True) LOG.warn('Handshaking: %s:%s, threadId:%s, id:%s', host, port, thread.get_ident(), id(self)) try: for callback in self._future_handshake_callbacks: if not is_coroutine_function(callback): raise SyntaxError( 'wrap generator function with ``@tornado.gen.coroutine`` first :%s ' % callback) LOG.debug('calling handshake_callback: %s', callback) r = yield callback(self) waiters = tuple(self._future_waiters) for fut in waiters: if not fut.done(): fut.set_result(self) except Exception as e: LOG.exception('Error Handshaking to %s:%s, retry.', host, port) self.future_stream.close() if not self._future_periodic_connect.is_running(): self._future_periodic_connect.start() return finally: self._future_connecting = False LOG.warn('Connected: %s:%s, threadId:%s, id:%s', host, port, thread.get_ident(), id(self)) try: for callback in self._future_initial_callbacks: if inspect.isgeneratorfunction(callback): raise SyntaxError( 'wrap generator function with ``@tornado.gen.coroutine`` first :%s ' % callback) LOG.debug('calling initial_callback: %s', callback) r = yield callback(self) except Exception as e: LOG.exception('Error connect to %s:%s, retry.', host, port) self.future_stream.close() if not self._future_periodic_connect.is_running(): self._future_periodic_connect.start() return for callback in self._future_close_callbacks: if inspect.isgeneratorfunction(callback): raise SyntaxError( 'wrap generator function with ``@tornado.gen.coroutine`` first :%s ' % callback) LOG.warn('Initialized: %s:%s, threadId:%s, id:%s!', host, port, thread.get_ident(), id(self))
def _inner(f): if not gen.is_coroutine_function(f): return wrapper(before_advices, around_advices, after_returning_advices, after_throwing_advices, after_advices)(f) @gen.coroutine @wraps(f) def _real_logic(*a, **kw): jp = JointPoint(f, a, kw) # 执行前置通知 for before_advice in before_advices: formal_arguments = inspect.getargspec(before_advice).args try: if "joint_point" in formal_arguments: return_value = before_advice(joint_point=jp) else: return_value = before_advice() if gen.is_future(return_value): yield return_value except Return as e: raise gen.Return(e.get_return_value()) # 执行环绕通知 around_after_advices = [] for around_advice in around_advices: formal_arguments = inspect.getargspec(around_advice).args try: if "joint_point" in formal_arguments: return_value = around_advice(joint_point=jp) else: return_value = around_advice() if gen.is_future(return_value): result = yield return_value else: result = return_value if callable(result): around_after_advices.append(result) except Return as e: raise gen.Return(e.get_return_value()) # 执行连接点 returning = None exc_info = None try: returning = jp.proceed() if gen.is_future(returning): returning = yield returning except: exc_info = sys.exc_info() # 执行环绕通知 for around_after_advice in around_after_advices: try: return_value = around_after_advice( jp, returning, exc_info) if gen.is_future(return_value): yield return_value except Return as e: raise gen.Return(e.get_return_value()) if exc_info is None: # 执行返回通知 for after_returning_advice in after_returning_advices: formal_arguments = inspect.getargspec( after_returning_advice).args kwargs = {} if "joint_point" in formal_arguments: kwargs["joint_point"] = jp if "returning" in formal_arguments: kwargs["returning"] = returning try: return_value = after_returning_advice(**kwargs) if gen.is_future(return_value): yield return_value except Return as e: raise gen.Return(e.get_return_value()) else: # 执行异常通知 for after_throwing_advice in after_throwing_advices: formal_arguments = inspect.getargspec( after_throwing_advice).args kwargs = {} if "joint_point" in formal_arguments: kwargs["joint_point"] = jp if "exc_info" in formal_arguments: kwargs["exc_info"] = exc_info try: return_value = after_throwing_advice(**kwargs) if gen.is_future(return_value): yield return_value except Return as e: raise gen.Return(e.get_return_value()) # 执行最终通知 for after_advice in after_advices: formal_arguments = inspect.getargspec(after_advice).args kwargs = {} if "joint_point" in formal_arguments: kwargs["joint_point"] = jp if "returning" in formal_arguments: kwargs["returning"] = returning if "exc_info" in formal_arguments: kwargs["exc_info"] = exc_info try: return_value = after_advice(**kwargs) if gen.is_future(return_value): yield return_value except Return as e: raise gen.Return(e.get_return_value()) if exc_info is not None: raise exc_info[0], exc_info[1], exc_info[2] raise gen.Return(returning) return _real_logic
def process(self, req): if gen.is_coroutine_function(self.predict_proba): ret = yield self.predict_proba(req.data) else: ret = self.predict_proba(req.data) req.ret = ret
def iscoroutinefunction(f): if gen.is_coroutine_function(f): return True if sys.version_info >= (3, 5) and inspect.iscoroutinefunction(f): return True return False
def test_retry(self): assert gen.is_coroutine_function(_retryable_coroutine) thing = NoIOErrorAfterCount(5) yield _retryable_coroutine(thing) assert thing.counter == thing.count
def iscoroutinefunction(f): if gen.is_coroutine_function(f): return True if sys.version_info >= (3, 5) and inspect.iscoroutinefunction(f): return True return False
def _iscoroutinefunction(f): # Python < 3.8 does not support determining if `partial` objects wrap async funcs if sys.version_info < (3, 8): while isinstance(f, functools.partial): f = f.func return inspect.iscoroutinefunction(f) or gen.is_coroutine_function(f)