def test_is_coroutine_function(self):
self.finished = True
def f():
yield gen.moment
coro = gen.coroutine(f)
self.assertFalse(gen.is_coroutine_function(f))
self.assertTrue(gen.is_coroutine_function(coro))
self.assertFalse(gen.is_coroutine_function(coro()))
def test_is_coroutine_function(self):
self.finished = True
def f():
yield gen.moment
coro = gen.coroutine(f)
self.assertFalse(gen.is_coroutine_function(f))
self.assertTrue(gen.is_coroutine_function(coro))
self.assertFalse(gen.is_coroutine_function(coro()))
def test_stop_after_attempt(self):
assert gen.is_coroutine_function(_retryable_coroutine)
thing = NoIOErrorAfterCount(2)
try:
yield _retryable_coroutine_with_2_attempts(thing)
except RetryError:
assert thing.counter == 2
def add_action(self, action: Union[Callable, Coroutine], interval: float, initial_delay: float = None) -> None:
"""
Add a new action
:param action: A function to call periodically
:param interval: The interval between execution of actions
:param initial_delay: Delay to the first execution, defaults to interval
"""
assert inspect.iscoroutinefunction(action) or gen.is_coroutine_function(action)
if self._stopped:
LOGGER.warning("Scheduling action '%s', while scheduler is stopped", action.__name__)
return
if initial_delay is None:
initial_delay = interval
LOGGER.debug("Scheduling action %s every %d seconds with initial delay %d", action, interval, initial_delay)
def action_function():
LOGGER.info("Calling %s" % action)
if action in self._scheduled:
try:
ensure_future_and_handle_exception(LOGGER, "Uncaught exception while executing scheduled action", action())
except Exception:
LOGGER.exception("Uncaught exception while executing scheduled action")
finally:
# next iteration
ihandle = IOLoop.current().call_later(interval, action_function)
self._scheduled[action] = ihandle
handle = IOLoop.current().call_later(initial_delay, action_function)
self._scheduled[action] = handle
def process_tasks(self):
""" Main task processing method (callback to register in ioloop). Consume and process tasks in queue
and reschedule processed tasks when relevant.
A task is processed if associated data was not removed from scheduler.
A task is rescheduled if processing callback returns False
(True means `task definitively done`) AND if task deadline is not null.
"""
while True:
task = yield self.tasks_queue.get() # type: _Task
try:
if task.valid and (not isinstance(task, _ImmediateTask)
or task.can_still_process()):
if gen.is_coroutine_function(self.callback_process):
remove_data = yield self.callback_process(task.data)
else:
remove_data = self.callback_process(task.data)
remove_data = remove_data or not task.deadline.delay
with (yield self.lock.acquire()):
del self.data_in_queue[task.data]
if not remove_data:
self.data_in_heap[task.data] = _Deadline(
self.current_time, task.deadline.delay)
finally:
self.tasks_queue.task_done()
def wrapped_callable(self, _callable, *args, **kw_args):
"""
wrap any kind of callable into chained callable, which will call itself after certain interval
"""
if is_coroutine_function(_callable):
@coroutine
def wrapper():
try:
yield _callable(*args, **kw_args)
except Exception as e:
app_log.exception(r'{}'.format(e))
if not self._stop:
yield sleep(self.interval)
loop = IOLoop.current()
loop.call_later(0, wrapper)
return wrapper
else:
def wrapper():
try:
_callable(*args, **kw_args)
except Exception as e:
app_log.exception(r'{}'.format(e))
if not self._stop:
loop = IOLoop.current()
loop.call_later(self.interval, wrapper)
return wrapper
def handle_request(server, request, connection_handler):
""" (coroutine) Find request handler function for associated request, run it and return its result.
:param server: a Server object to pass to handler function.
:param request: a request object to pass to handler function.
See diplomacy.communication.requests for possible requests.
:param connection_handler: a ConnectionHandler object to pass to handler function.
:return: (future) either None or a response object.
See module diplomacy.communication.responses for possible responses.
"""
request_handler_fn = MAPPING.get(type(request), None)
if not request_handler_fn:
raise exceptions.RequestException()
game = server.get_game(request.game_id)
# Game not found
if not game or game.is_game_completed or game.is_game_canceled:
future = Future()
future.set_result([responses.REJ(bytes(request))])
return future
if gen.is_coroutine_function(request_handler_fn):
# Throw the future returned by this coroutine.
return request_handler_fn(server, request, connection_handler, game)
# Create and return a future.
future = Future()
try:
result = request_handler_fn(server, request, connection_handler, game)
future.set_result(result)
except exceptions.DiplomacyException as exc:
future.set_exception(exc)
return future
def run_async(func, *args, **kwargs):
if not inspect.iscoroutinefunction(func) and not gen.is_coroutine_function(
func):
fu = registry.executor_pool().submit(func, *args, **kwargs)
fu = to_tornado_future(fu)
else:
fu = func(*args, **kwargs)
return gen.convert_yielded(fu)
def has_keyword(func, keyword):
if PY3:
return keyword in inspect.signature(func).parameters
else:
# https://stackoverflow.com/questions/50100498/determine-keywords-of-a-tornado-coroutine
if gen.is_coroutine_function(func):
func = func.__wrapped__
return keyword in inspect.getargspec(func).args
def has_keyword(func, keyword):
if PY3:
return keyword in inspect.signature(func).parameters
else:
# https://stackoverflow.com/questions/50100498/determine-keywords-of-a-tornado-coroutine
if gen.is_coroutine_function(func):
func = func.__wrapped__
return keyword in inspect.getargspec(func).args
async def wrapper(self, *args, **kwargs):
spec = getfullargspec(method)
filling_args = spec.args[len(args) + 1:] # 切出需要填充的参数
default_values = spec.defaults[-len(filling_args):] if spec.defaults else [] # 切出需要的默认值
# 倒序，参数与默认值对齐
for key, default in zip_longest(reversed(filling_args), reversed(default_values)):
if key in kwargs:
continue
if key in self._json_args:
kwargs[key] = self._json_args.get(key)
continue
if isinstance(default, list):
value = self.get_arguments(key, True) or default
else:
value = self.get_argument(key, default)
kwargs[key] = value
# 根据注解做类型转换
model_dict = {}
for key, value in kwargs.items():
if key not in spec.annotations:
continue
annotations = spec.annotations.get(key)
try:
if issubclass(annotations, BaseModel):
model = annotations(context=HandlerContext(self))
kwargs[key] = model
model_dict[key] = model
elif isinstance(value, list):
kwargs[key] = [annotations(item) for item in value]
elif value:
kwargs[key] = annotations(value)
except Exception as e:
logging.exception(e)
logging.info(f'{key} 字段期待类型为: {str(annotations)} 实际为: "{value}"')
self.finish({'code': -1, 'msg': '参数错误'})
return
try: # 捕获异常，关闭连接
if iscoroutinefunction(method) or is_coroutine_function(method):
response = await method(self, *args, **kwargs)
else:
response = method(self, *args, **kwargs)
return response
except ParametersError as e:
logging.exception(e)
self.finish({'code': -1, 'msg': '参数错误'})
except (NotFound, FileTypeError, Duplicate, PermissionDenied, InternalError) as e:
self.finish({'code': e.code, 'msg': e.msg})
except Finish as e:
raise e
except Exception as e:
logging.exception(e)
self.finish({'code': -1, 'msg': '内部错误'})
finally:
for key, model in model_dict.items():
model.clear()
def wrapper(*args, **kwargs):
func_result = yield threadpool_executor.submit(func, *args, **kwargs)
if is_coroutine_function(func):
result = yield func_result
else:
result = func_result
return result
def addCleanup(self, function, *args, **kwargs):
"""
Add a function, with arguments, to be called when the test is
completed. If function is a coroutine function, it will run on the loop
before it's cleaned.
"""
if gen.is_coroutine_function(function):
return self._async_cleanups.append((function, args, kwargs))
return super(BaseTestCase, self).addCleanup(function, *args, **kwargs)
def delete(self, path):
path = path.strip('/')
self.log.info('Appmode deleting: %s', path)
# delete session, including the kernel
sm = self.session_manager
if gen.is_coroutine_function(sm.get_session):
s = yield sm.get_session(path=path)
else:
s = sm.get_session(path=path)
if gen.is_coroutine_function(sm.delete_session):
yield sm.delete_session(session_id=s['id'])
else:
sm.delete_session(session_id=s['id'])
# delete tmp copy
cm = self.contents_manager
cm.delete(path)
self.finish()
def ensure_coroutine(func_or_coro):
if gen.is_coroutine_function(func_or_coro):
return func_or_coro
else:
@coroutine
def coro(*args, **kwargs):
raise Return(func_or_coro(*args, **kwargs))
return coro
def post(self, path):
assert self.get_body_arguments("appmode_action")[0] == "delete"
path = path.strip('/')
self.log.info('Appmode deleting: %s', path)
# delete session, including the kernel
sm = self.session_manager
if gen.is_coroutine_function(sm.get_session):
s = yield sm.get_session(path=path)
else:
s = sm.get_session(path=path)
if gen.is_coroutine_function(sm.delete_session):
yield sm.delete_session(session_id=s['id'])
else:
sm.delete_session(session_id=s['id'])
# delete tmp copy
cm = self.contents_manager
cm.delete(path)
self.finish()
def add_action(
self,
action: TaskMethod,
interval: float,
initial_delay: Optional[float] = None,
) -> None:
"""
Add a new action
:param action: A function to call periodically
:param interval: The interval between execution of actions
:param initial_delay: Delay to the first execution, defaults to interval
"""
assert inspect.iscoroutinefunction(
action) or gen.is_coroutine_function(action)
if self._stopped:
LOGGER.warning(
"Scheduling action '%s', while scheduler is stopped",
action.__name__)
return
if initial_delay is None:
initial_delay = interval
LOGGER.debug(
"Scheduling action %s every %d seconds with initial delay %d",
action, interval, initial_delay)
def action_function() -> None:
LOGGER.info("Calling %s" % action)
if action in self._scheduled:
try:
task = ensure_future_and_handle_exception(
logger=LOGGER,
msg=
"Uncaught exception while executing scheduled action",
action=action(),
notify_done_callback=functools.partial(
self._notify_done, action),
)
self._add_to_executing_tasks(action, task)
except Exception:
LOGGER.exception(
"Uncaught exception while executing scheduled action")
finally:
# next iteration
ihandle = IOLoop.current().call_later(
interval, action_function)
self._scheduled[action] = ihandle
handle = IOLoop.current().call_later(initial_delay, action_function)
self._scheduled[action] = handle
def start_io_loop(on_start_callback, custom_io_loop=None, stop_on_interrupt=True, **kwargs):
""" Starts an asynchronous IO Loop
:param on_start_callback: The callback to call when the loop is started.
:param custom_io_loop: A custom IO loop object to start, otherwise the .instance() loop will be started.
:param stop_on_interrupt: Boolean. If true, the io loop will stop on CTRL-C.
:return: Nothing
"""
if not is_coroutine_function(on_start_callback):
LOGGER.error('The on_start_callback must be a gen.coroutine()')
raise RuntimeError()
io_loop = custom_io_loop or ioloop.IOLoop.instance()
io_loop.spawn_callback(on_start_callback, **kwargs)
try:
io_loop.start()
except KeyboardInterrupt:
if stop_on_interrupt:
io_loop.stop()
def iscoroutinepartial(coro):
"""
Function returns True if function it's a partial instance of coroutine. See additional information here_.
:param coro: Function
:return: bool
.. _here: https://goo.gl/C0S4sQ
"""
while True:
parent = coro
coro = getattr(parent, 'func', None)
if coro is None:
break
return gen.is_coroutine_function(parent)
def submit_task(self, function, *args, **kwargs):
async_result = AsyncResult()
if self._shutting_down or self._shuted_down:
async_result.set_exception(
ShutedDownError(self._coroutine_pool_name))
return async_result
if not gen.is_coroutine_function(function):
async_result.set_exception(
RuntimeError("function must be tornado coroutine function"))
return async_result
is_full = False
task_item = TaskItem(function, args, kwargs, async_result)
try:
self._queue.put_nowait(task_item)
async_result.set_time_info("submitted_to_queue_at")
except QueueFull:
is_full = True
if is_full:
return self._reject_handler(self._queue, task_item)
else:
self._core_coroutines_wait_condition.notify()
return async_result
def read_stream(self):
""" Read the next message from the stream """
messages = []
in_message = yield DaideMessage.from_stream(self.stream)
if in_message and in_message.is_valid:
message_handler = self.message_mapping.get(in_message.message_type,
None)
if not message_handler:
raise RuntimeError(
'Unrecognized DAIDE message type [{}]'.format(
in_message.message_type))
if gen.is_coroutine_function(message_handler):
messages = yield message_handler(in_message)
else:
messages = message_handler(in_message)
elif in_message:
err_message = ErrorMessage()
err_message.error_code = in_message.error_code
messages = [err_message]
for message in messages:
yield self.write_message(message)
def is_coroutine_function(f):
return asyncio.iscoroutinefunction(f) or gen.is_coroutine_function(f)
def iscoroutinefunction(f):
return inspect.iscoroutinefunction(f) or gen.is_coroutine_function(f)
def test_retry(self):
assert gen.is_coroutine_function(_retryable_coroutine)
thing = NoIOErrorAfterCount(5)
yield _retryable_coroutine(thing)
assert thing.counter == thing.count
def _future_stream_connect(self,
host=None,
port=None,
af=socket.AF_UNSPEC,
ssl_options=None,
max_buffer_size=None,
source_ip=None,
source_port=None,
timeout=None):
assert self._future_thread_id == thread.get_ident(
), 'this should run in IO thread:%s' % self._future_thread_id
if self.future_connected():
if self._future_periodic_connect.is_running():
self._future_periodic_connect.stop()
raise Return(self)
if self._future_connecting is True:
return
else:
self._future_connecting = True
host = host or self.host
port = port or self.port
LOG.warn('Connecting %s:%s, threadId:%s, id:%s' %
(host, port, thread.get_ident(), id(self)))
try:
self.future_stream = yield self._future_connector.connect(
host,
port,
timeout=timeout,
max_buffer_size=max_buffer_size,
af=af,
ssl_options=ssl_options,
source_ip=source_ip,
source_port=source_port)
# Note: we first register Async Close|Read|Write callback here, since handshake may require Async READ|WRITE
except Exception as e:
self._future_connecting = False
LOG.exception('Error connect to %s:%s, retry.', host, port)
if self.future_stream:
self.future_stream.close()
if not self._future_periodic_connect.is_running():
self._future_periodic_connect.start()
return
self._future_callback_queue.clear() # registry
self._future_packet_bulk = []
self._future_connecting = False
self._future_chunk_size = stream_options(self.future_stream,
blocking=False,
keepalive=True)
self._future_reader_buffer = bytearray(self._future_chunk_size)
self.future_stream.set_close_callback(self._future_stream_close)
self.future_stream.read_into(self._future_reader_buffer,
callback=self._future_socket_read,
partial=True)
LOG.warn('Handshaking: %s:%s, threadId:%s, id:%s', host, port,
thread.get_ident(), id(self))
try:
for callback in self._future_handshake_callbacks:
if not is_coroutine_function(callback):
raise SyntaxError(
'wrap generator function with ``@tornado.gen.coroutine`` first :%s '
% callback)
LOG.debug('calling handshake_callback: %s', callback)
r = yield callback(self)
waiters = tuple(self._future_waiters)
for fut in waiters:
if not fut.done():
fut.set_result(self)
except Exception as e:
LOG.exception('Error Handshaking to %s:%s, retry.', host, port)
self.future_stream.close()
if not self._future_periodic_connect.is_running():
self._future_periodic_connect.start()
return
finally:
self._future_connecting = False
LOG.warn('Connected: %s:%s, threadId:%s, id:%s', host, port,
thread.get_ident(), id(self))
try:
for callback in self._future_initial_callbacks:
if inspect.isgeneratorfunction(callback):
raise SyntaxError(
'wrap generator function with ``@tornado.gen.coroutine`` first :%s '
% callback)
LOG.debug('calling initial_callback: %s', callback)
r = yield callback(self)
except Exception as e:
LOG.exception('Error connect to %s:%s, retry.', host, port)
self.future_stream.close()
if not self._future_periodic_connect.is_running():
self._future_periodic_connect.start()
return
for callback in self._future_close_callbacks:
if inspect.isgeneratorfunction(callback):
raise SyntaxError(
'wrap generator function with ``@tornado.gen.coroutine`` first :%s '
% callback)
LOG.warn('Initialized: %s:%s, threadId:%s, id:%s!', host, port,
thread.get_ident(), id(self))
def _inner(f):
if not gen.is_coroutine_function(f):
return wrapper(before_advices, around_advices,
after_returning_advices, after_throwing_advices,
after_advices)(f)
@gen.coroutine
@wraps(f)
def _real_logic(*a, **kw):
jp = JointPoint(f, a, kw)
# 执行前置通知
for before_advice in before_advices:
formal_arguments = inspect.getargspec(before_advice).args
try:
if "joint_point" in formal_arguments:
return_value = before_advice(joint_point=jp)
else:
return_value = before_advice()
if gen.is_future(return_value):
yield return_value
except Return as e:
raise gen.Return(e.get_return_value())
# 执行环绕通知
around_after_advices = []
for around_advice in around_advices:
formal_arguments = inspect.getargspec(around_advice).args
try:
if "joint_point" in formal_arguments:
return_value = around_advice(joint_point=jp)
else:
return_value = around_advice()
if gen.is_future(return_value):
result = yield return_value
else:
result = return_value
if callable(result):
around_after_advices.append(result)
except Return as e:
raise gen.Return(e.get_return_value())
# 执行连接点
returning = None
exc_info = None
try:
returning = jp.proceed()
if gen.is_future(returning):
returning = yield returning
except:
exc_info = sys.exc_info()
# 执行环绕通知
for around_after_advice in around_after_advices:
try:
return_value = around_after_advice(
jp, returning, exc_info)
if gen.is_future(return_value):
yield return_value
except Return as e:
raise gen.Return(e.get_return_value())
if exc_info is None:
# 执行返回通知
for after_returning_advice in after_returning_advices:
formal_arguments = inspect.getargspec(
after_returning_advice).args
kwargs = {}
if "joint_point" in formal_arguments:
kwargs["joint_point"] = jp
if "returning" in formal_arguments:
kwargs["returning"] = returning
try:
return_value = after_returning_advice(**kwargs)
if gen.is_future(return_value):
yield return_value
except Return as e:
raise gen.Return(e.get_return_value())
else:
# 执行异常通知
for after_throwing_advice in after_throwing_advices:
formal_arguments = inspect.getargspec(
after_throwing_advice).args
kwargs = {}
if "joint_point" in formal_arguments:
kwargs["joint_point"] = jp
if "exc_info" in formal_arguments:
kwargs["exc_info"] = exc_info
try:
return_value = after_throwing_advice(**kwargs)
if gen.is_future(return_value):
yield return_value
except Return as e:
raise gen.Return(e.get_return_value())
# 执行最终通知
for after_advice in after_advices:
formal_arguments = inspect.getargspec(after_advice).args
kwargs = {}
if "joint_point" in formal_arguments:
kwargs["joint_point"] = jp
if "returning" in formal_arguments:
kwargs["returning"] = returning
if "exc_info" in formal_arguments:
kwargs["exc_info"] = exc_info
try:
return_value = after_advice(**kwargs)
if gen.is_future(return_value):
yield return_value
except Return as e:
raise gen.Return(e.get_return_value())
if exc_info is not None:
raise exc_info[0], exc_info[1], exc_info[2]
raise gen.Return(returning)
return _real_logic
def process(self, req):
if gen.is_coroutine_function(self.predict_proba):
ret = yield self.predict_proba(req.data)
else:
ret = self.predict_proba(req.data)
req.ret = ret
def iscoroutinefunction(f):
if gen.is_coroutine_function(f):
return True
if sys.version_info >= (3, 5) and inspect.iscoroutinefunction(f):
return True
return False
def test_retry(self):
assert gen.is_coroutine_function(_retryable_coroutine)
thing = NoIOErrorAfterCount(5)
yield _retryable_coroutine(thing)
assert thing.counter == thing.count
def iscoroutinefunction(f):
if gen.is_coroutine_function(f):
return True
if sys.version_info >= (3, 5) and inspect.iscoroutinefunction(f):
return True
return False
def _iscoroutinefunction(f):
# Python < 3.8 does not support determining if `partial` objects wrap async funcs
if sys.version_info < (3, 8):
while isinstance(f, functools.partial):
f = f.func
return inspect.iscoroutinefunction(f) or gen.is_coroutine_function(f)