async def main(f: asyncio.Future):
await asyncio.sleep(1)
try:
f.set_result('I have finished')
except RuntimeError as e:
print(f'No longer allowed: {e}')
f.cancel()
async def greet(q: asyncio.Queue, reader: asyncio.Future = None):
"""
loop: asyncio.AbstractEventLoop
:param q:
:return:
"""
while True:
name = await q.get()
if not name:
print('][break][')
break
if name == 'q' or name == 'quit':
print('][quit][')
break
print('salvete, {}'.format(name), flush=True)
# await asyncio.sleep(0.1)
print('___', flush=True)
# await proc.wait()
print('[terminating]')
if reader:
reader.cancel()
print('[reader cancelled]')
async def greet(q: asyncio.Queue, reader: asyncio.Future):
"""
loop: asyncio.AbstractEventLoop
:param q:
:return:
"""
# proc = await asyncio.subprocess.create_subprocess_exec(
# './hello.py', stdin=asyncio.subprocess.PIPE,
# stdout=asyncio.subprocess.PIPE
# )
proc = await asyncio.subprocess.create_subprocess_shell(
'./hello.py --pipe',
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE)
while True:
name = await q.get()
print('<sending {}>'.format(name), flush=True)
proc.stdin.write('{}\n'.format(name).encode())
res = await proc.stdout.read(1024)
if not res:
print('<break>')
break
print(res.decode(), flush=True)
# await asyncio.sleep(0.1)
await proc.wait()
print('<done waiting>')
# tasks = asyncio.tasks
# loop.close()
# print(tasks)
reader.cancel()
print('<reader cancelled>')
async def main2(f: asyncio.Future):
await asyncio.sleep(1.0)
try:
f.set_result("I have finished.")
except RuntimeError as e:
print("No longer allowed: ", e)
f.cancel()
def run_forever(self, sub_keepalive: asyncio.Future) -> None:
"""
Start the asyncio loop, running until it is either SIGTERM-ed or killed
by keyboard interrupt. The Future parameter is used to cancel
subscription Future in the case that an unexpected exception is thrown.
You can also directly pass the `.subscribe()` method call instead like
so:
sub.run_forever(sub.subscribe(callback))
"""
try:
self.loop.run_forever()
except (KeyboardInterrupt, concurrent.futures.CancelledError):
pass
finally:
# 1. stop the `SubscriberClient` future, which will prevent more
# tasks from being leased
if not sub_keepalive.cancelled():
sub_keepalive.cancel()
# 2. cancel the tasks we already have, which should just be
# `worker` instances; note they have
# `except CancelledError: pass`
for task in asyncio.Task.all_tasks(loop=self.loop):
task.cancel()
# 3. stop the `asyncio` event loop
self.loop.stop()
def _stop_task(key: str, future: asyncio.Future) -> None:
if not future.cancelled():
future.cancel()
logging.info(f"stop task:{key}")
elif future.done():
logging.warning(f"task:{key} already stop")
else:
logging.warning(f"{key} can't stop")
async def cancel(fut: asyncio.Future) -> None:
"""
Cancel a future/task and await for it to cancel.
This method suppresses the CancelledError
"""
fut.cancel()
await asyncio.sleep(0) # let loop cycle
with suppress(asyncio.CancelledError):
await fut
class Stream:
def __init__(self):
self._future = Future()
self._complete = Event()
self._listeners = [ ]
def __aiter__(self):
return self
async def __anext__(self):
if self._complete.is_set():
raise StopAsyncIteration
result = await self._future
self._future = Future()
return result
def write(self, item, last):
if self._complete.is_set():
return
if last:
self._set_complete()
if self._future.done():
self._future = Future()
self._future.set_result(item)
def abort(self, exc):
if self._complete.is_set():
return
self._set_complete()
if self._future.done():
self._future = Future()
self._future.set_exception(exc)
def cancel(self):
self._set_complete()
if self._future.done():
self._future = Future()
self._future.cancel()
async def completed(self):
await self._complete.wait()
@property
def is_complete(self):
return self._complete.is_set()
def add_complete_listener(self, listener):
self._listeners.append(listener)
def _set_complete(self):
self._complete.set()
for listener in self._listeners:
listener()
def _finalise_future(fut: Future) -> Optional[Union[Response, str]]:
if fut.done():
err = fut.exception()
if err:
error_logger.error("%s", err, exc_info=err)
else:
return fut.result()
else:
fut.cancel()
def _done_callback(future: asyncio.Future, task: asyncio.Task):
if task.cancelled():
future.cancel()
return
exception = task.exception()
if exception is not None:
future.set_exception(exception)
return
future.set_result(task.result())
async def wait_for_future(self, future: asyncio.Future) -> Any:
done, _ = await asyncio.wait(set([future, *self._failures]),
return_when=asyncio.FIRST_COMPLETED)
if future not in done:
future.cancel()
for failure in self._failures:
if failure not in done:
failure.cancel()
for failure in self._failures:
if failure in done:
raise failure.result()
return future.result()
async def connect(self, disconnect_future: asyncio.Future = None):
self.loop = asyncio.get_event_loop()
client_logger.info("Establishing connection to %s:%d", self.address, self.port)
try:
self._transport, self._protocol = \
await self.loop.create_connection(lambda: ClientProtocol(self, disconnect_future),
self.address, self.port)
except OSError as e:
client_logger.warning("Failed to establish connection:")
client_logger.warning("%s", str(e))
disconnect_future.cancel()
raise
client_logger.info("Established connection to %s:%d", self.address, self.port)
def go():
future = Future()
source = rx.from_future(future)
def on_next(x):
success[0] = False
def on_error(err):
success[1] = type(err) == asyncio.CancelledError
def on_completed():
success[2] = False
source.subscribe(on_next, on_error, on_completed)
future.cancel()
def copy_result(from_: Future, to: Future) -> None:
if not from_.done():
raise ValueError('from_ must be a completed Future')
if to.done():
raise ValueError('to must NOT be a completed Future')
if from_.cancelled():
to.cancel()
else:
exception = from_.exception()
if exception is not None:
to.set_exception(exception)
else:
result = from_.result()
to.set_result(result)
def reject_on(self, future: asyncio.Future, error: Error) -> None:
async def future_wrapper() -> Error:
await future
return error
result = self._loop.create_task(future_wrapper())
result.add_done_callback(lambda f: future.cancel())
self._failures.append(result)
def get(self):
"""Remove and return an item from the channel.
If channel is empty, wait until an item is available.
This method is a coroutine.
"""
while self.empty() and not self._close.is_set():
getter = Future(loop=self._loop)
self._getters.append(getter)
try:
yield from getter
except ChannelClosed:
raise
except:
getter.cancel() # Just in case getter is not done yet.
if not self.empty() and not getter.cancelled():
# We were woken up by put_nowait(), but can't take
# the call. Wake up the next in line.
self._wakeup_next(self._getters)
raise
return self.get_nowait()
def _done_callback(fut: Future) -> None:
try:
fut.result() # try raise exception
except CancelledError:
fut.cancel()
nonlocal finished, result, _future
finished += 1
if _future is None:
_future = fut
for task in tasks:
if task.done() or task.cancelled():
continue
task.cancel()
if finished == len(tasks):
result.set_result(_future.result())
async def wait_with_cancellation(
self, runner_task: asyncio.Future) -> TestRunStatus:
try:
# Here we wait for the first event, which may be:
# 1. Watchdog termination signal
# 2. Test runner events
# 2.1 Run to end
# 2.2 Timeout during run
waited = await asyncio.wait(
[termination_event.wait(), runner_task],
return_when=asyncio.FIRST_COMPLETED)
if runner_task.done():
return TestRunStatus('success', runner_task.result())
if termination_event.is_set():
logger.warning('Looks like task was cancelled by user...')
runner_task.cancel()
return TestRunStatus('terminated')
except asyncio.TimeoutError:
logger.warning('Timeout reached while waiting for tests...')
return TestRunStatus('timeout')
def put(self, item):
"""Put an item into the channel.
If the channel is full, wait until a free
slot is available before adding item.
If the channel is closed or closing, raise ChannelClosed.
This method is a coroutine.
"""
while self.full() and not self._close.is_set():
putter = Future(loop=self._loop)
self._putters.append(putter)
try:
yield from putter
except ChannelClosed:
raise
except:
putter.cancel() # Just in case putter is not done yet.
if not self.full() and not putter.cancelled():
# We were woken up by get_nowait(), but can't take
# the call. Wake up the next in line.
self._wakeup_next(self._putters)
raise
return self.put_nowait(item)
class ReusableFuture:
def __init__(self):
self.impl = Future()
self.waiting = False
def __await__(self):
self.waiting = True
try:
result = yield from self.impl
return result
finally:
self.waiting = False
self.impl = Future()
def cancel(self):
if self.waiting:
self.impl.cancel()
def set_result(self, value):
if self.waiting:
self.impl.set_result(value)
async def _async_poll_for_reply(
self,
msg_id: str,
cell: NotebookNode,
timeout: t.Optional[int],
task_poll_output_msg: asyncio.Future,
task_poll_kernel_alive: asyncio.Future,
) -> t.Dict:
assert self.kc is not None
new_timeout: t.Optional[float] = None
if timeout is not None:
deadline = monotonic() + timeout
new_timeout = float(timeout)
while True:
try:
msg = await ensure_async(
self.kc.shell_channel.get_msg(timeout=new_timeout))
if msg['parent_header'].get('msg_id') == msg_id:
if self.record_timing:
cell['metadata']['execution'][
'shell.execute_reply'] = timestamp()
try:
await asyncio.wait_for(task_poll_output_msg,
self.iopub_timeout)
except (asyncio.TimeoutError, Empty):
if self.raise_on_iopub_timeout:
task_poll_kernel_alive.cancel()
raise CellTimeoutError.error_from_timeout_and_cell(
"Timeout waiting for IOPub output",
self.iopub_timeout, cell)
else:
self.log.warning(
"Timeout waiting for IOPub output")
task_poll_kernel_alive.cancel()
return msg
else:
if new_timeout is not None:
new_timeout = max(0, deadline - monotonic())
except Empty:
# received no message, check if kernel is still alive
assert timeout is not None
task_poll_kernel_alive.cancel()
await self._async_check_alive()
await self._async_handle_timeout(timeout, cell)
async def _poll_for_reply(
self,
msg_id: str,
timeout: t.Optional[int],
task_poll_output_msg: asyncio.Future,
task_poll_kernel_alive: asyncio.Future,
) -> t.Dict:
assert self.kc is not None
new_timeout: t.Optional[float] = None
if timeout is not None:
deadline = monotonic() + timeout
new_timeout = float(timeout)
while True:
try:
msg = await ensure_async(self.kc.shell_channel.get_msg(timeout=new_timeout))
if msg["parent_header"].get("msg_id") == msg_id:
try:
await asyncio.wait_for(task_poll_output_msg, self.iopub_timeout)
except (asyncio.TimeoutError, Empty):
if self.raise_on_iopub_timeout:
task_poll_kernel_alive.cancel()
raise ExecTimeoutError("Timeout waiting for IOPub output")
else:
self.log.warning("Timeout waiting for IOPub output")
task_poll_kernel_alive.cancel()
return msg
else:
if new_timeout is not None:
new_timeout = max(0, deadline - monotonic())
except Empty:
# received no message, check if kernel is still alive
assert timeout is not None
task_poll_kernel_alive.cancel()
await self._check_alive()
await self._handle_timeout(timeout)
def maybe_cancel(fut: asyncio.Future) -> bool:
"""Cancel future if it is cancellable."""
if fut is not None and not fut.done():
return fut.cancel()
return False
async def cancel_task(task: asyncio.Future) -> None:
task.cancel()
await forever
class Bot:
__slots__ = ("logger", "loop", "values", "server", "main_task",
"longpoll_request", "settings", "api", "handler",
"logger_file")
def __init__(self,
settings,
logger=None,
handler=None,
loop=asyncio.get_event_loop()):
self.logger = None
self.init_logger(logger)
self.logger.info("Initializing bot")
self.loop = loop
self.values = {}
self.server = ""
self.longpoll_request = None
self.main_task = None
self.settings = settings
self.logger.info("Initializing vk clients")
self.api = VkController(settings, logger=self.logger)
self.logger.info("Loading plugins")
if handler:
self.handler = handler
else:
self.handler = MessageHandler(self,
self.api,
initiate_plugins=False)
self.handler.initiate_plugins()
signal.signal(signal.SIGINT, lambda x, y: self.stop_bot(True))
self.logger.info("Bot succesfully initialized")
def init_logger(self, logger):
if not logger:
logger = logging.Logger("sketal", level=logging.INFO)
formatter = logging.Formatter(
fmt=u'%(filename)-10s [%(asctime)s] %(levelname)-8s: %(message)s',
datefmt='%y.%m.%d %H:%M:%S')
file_handler = logging.FileHandler('logs.txt')
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
self.logger_file = file_handler
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
stream_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
self.logger = logger
async def init_long_polling(self, update=0):
result = None
retries = 10
for x in range(retries):
result = await self.api(sender=self.api.target_client
).messages.getLongPollServer(use_ssl=1,
lp_version=2)
if result:
break
time.sleep(0.5)
if not result:
self.logger.error("Unable to connect to VK's long polling server")
exit()
last_ts = 0
longpoll_key = ""
if 'ts' in self.values:
last_ts = self.values['ts']
if 'key' in self.values:
longpoll_key = self.values['key']
if update == 0:
self.server = "https://" + result['server']
longpoll_key = result['key']
last_ts = result['ts']
elif update == 3:
longpoll_key = result['key']
last_ts = result['ts']
elif update == 2:
longpoll_key = result['key']
self.values = {
'act': 'a_check',
'key': longpoll_key,
'ts': last_ts,
'wait': 20,
'mode': 10,
'version': 2
}
async def process_longpoll_event(self, new_event):
if not new_event:
return
event_id = new_event[0]
if event_id != 4:
evnt = LongpollEvent(self.api, event_id, new_event)
return await self.process_event(evnt)
data = MessageEventData()
data.msg_id = new_event[1]
data.attaches = new_event[6]
data.time = int(new_event[4])
try:
data.user_id = int(data.attaches['from'])
data.chat_id = int(new_event[3]) - 2000000000
data.is_multichat = True
del data.attaches['from']
except KeyError:
data.user_id = int(new_event[3])
data.is_multichat = False
# https://vk.com/dev/using_longpoll_2
flags = parse_msg_flags(new_event[2])
if flags['outbox']:
if not self.settings.READ_OUT:
return
data.is_out = True
data.full_text = new_event[5].replace('<br>', '\n')
if "fwd" in data.attaches:
data.forwarded = MessageEventData.parse_brief_forwarded_messages_from_lp(
data.attaches["fwd"])
del data.attaches["fwd"]
else:
data.forwarded = []
msg = Message(self.api, data)
if await self.check_event(data.user_id, data.chat_id, data.attaches):
msg.is_event = True
await self.process_message(msg)
async def longpoll_processor(self):
await self.init_long_polling()
session = aiohttp.ClientSession(loop=self.loop)
while True:
try:
self.longpoll_request = session.get(self.server,
params=self.values)
resp = await self.longpoll_request
except aiohttp.ClientOSError:
session = aiohttp.ClientSession(loop=self.loop)
except (asyncio.TimeoutError, aiohttp.ServerDisconnectedError):
self.logger.warning(
"Long polling server doesn't respond. Changing server")
await self.init_long_polling()
continue
try:
events = json.loads(await resp.text())
except ValueError:
continue
failed = events.get('failed')
if failed:
err_num = int(failed)
if err_num == 1: # 1 - update timestamp
self.values['ts'] = events['ts']
elif err_num in (2, 3): # 2, 3 - new data for long polling
await self.init_long_polling(err_num)
continue
self.values['ts'] = events['ts']
for event in events['updates']:
asyncio.ensure_future(self.process_longpoll_event(event))
async def callback_processor(self, request):
try:
data = await request.json()
except (UnicodeDecodeError, json.decoder.JSONDecodeError):
return web.Response(text="ok")
data_type = data["type"]
if data_type == "confirmation":
return web.Response(text=self.settings.CONF_CODE)
obj = data["object"]
if "user_id" in obj:
obj['user_id'] = int(obj['user_id'])
if data_type == 'message_new':
data = MessageEventData.from_message_body(obj)
msg = Message(self.api, data)
await self.process_message(msg)
else:
evnt = CallbackEvent(self.api, data_type, obj)
await self.process_event(evnt)
return web.Response(text="ok")
def longpoll_run(self, custom_process=False):
self.main_task = Task(self.longpoll_processor())
if custom_process:
return self.main_task
self.logger.info("Started to process messages")
try:
self.loop.run_until_complete(self.main_task)
except (KeyboardInterrupt, SystemExit):
self.stop()
self.logger.info("Stopped to process messages")
except asyncio.CancelledError:
pass
def callback_run(self, custom_process=False):
host = getenv('IP', '0.0.0.0')
port = int(getenv('PORT', 8000))
self.logger.info("Started to process messages")
try:
server_generator, handler, app = self.loop.run_until_complete(
self.init_app(host, port, self.loop))
server = self.loop.run_until_complete(server_generator)
except OSError:
self.logger.error("Address already in use: " + str(host) + ":" +
str(port))
return
self.main_task = Future()
if custom_process:
return self.main_task
print("======== Running on http://{}:{} ========\n"
" (Press CTRL+C to quit)".format(
*server.sockets[0].getsockname()))
def stop_server():
server.close()
if not self.loop.is_running():
return
self.loop.run_until_complete(server.wait_closed())
self.loop.run_until_complete(app.shutdown())
self.loop.run_until_complete(handler.shutdown(10))
self.loop.run_until_complete(app.cleanup())
try:
self.loop.run_until_complete(self.main_task)
except KeyboardInterrupt:
self.stop()
stop_server()
self.loop.close()
except asyncio.CancelledError:
pass
finally:
stop_server()
self.logger.info("Stopped to process messages")
async def init_app(self, host, port, loop):
app = web.Application()
app.router.add_post('/', self.callback_processor)
handler = app.make_handler()
server_generator = loop.create_server(handler, host, port)
return server_generator, handler, app
def stop_bot(self, full=False):
try:
self.main_task.cancel()
except:
pass
if full:
self.stop()
self.loop.stop()
self.logger.info("Attempting to turn bot off")
async def process_message(self, msg):
asyncio.ensure_future(self.handler.process(msg), loop=self.loop)
async def check_event(self, user_id, chat_id, attaches):
if chat_id != 0 and "source_act" in attaches:
photo = attaches.get("attach1_type") + attaches.get(
"attach1") if "attach1" in attaches else None
evnt = ChatChangeEvent(self.api, user_id, chat_id,
attaches.get("source_act"),
int(attaches.get("source_mid", 0)),
attaches.get("source_text"),
attaches.get("source_old_text"), photo,
int(attaches.get("from", 0)))
await self.process_event(evnt)
return True
return False
async def process_event(self, evnt):
asyncio.ensure_future(self.handler.process_event(evnt), loop=self.loop)
def do(self, coroutine):
if asyncio.iscoroutine(coroutine):
return self.loop.run_until_complete(coroutine)
return False
@staticmethod
def silent(func):
try:
func()
except:
pass
def stop(self):
self.handler.stop()
self.api.stop()
self.silent(self.main_task.cancel)
self.logger.removeHandler(self.logger_file)
self.logger_file.close()
async def kill_timeout(self, future: asyncio.Future):
asyncio.sleep(self._timeout)
future.cancel()
async def _stop_subscription(cls, future: asyncio.Future) -> None:
future.cancel()
await future
result = future.result()
await result.aclose()
def _close_app(app: Application, fut: asyncio.Future) -> None:
if not fut.done():
logger.info("Cancelling consumer from signal")
fut.cancel()
async def _wait_timeout(self, future: asyncio.Future):
await asyncio.sleep(self._timeout)
future.cancel()
_logger.info('timeout to connect server')
async def _timeout(future: asyncio.Future, time: float) -> None:
await asyncio.sleep(time)
future.cancel()
def maybe_cancel(fut: asyncio.Future) -> bool:
if fut is not None and not fut.done():
return fut.cancel()
return False
async def _rp_task_watcher(
task: rp.Task, # noqa: C901
future: asyncio.Future,
final: RPFinalTaskState,
ready: asyncio.Event) -> rp.Task:
"""Manage the relationship between an RP.Task and a scalems Future.
Cancel the RP.Task if this task or the scalems.Future is canceled.
Publish the RP.Task result or cancel the scalems.Future if the RP.Task is
done or canceled.
Arguments:
task: RADICAL Pilot Task, submitted by caller.
future: asyncio.Future to which *task* results should be propagated.
ready: output parameter, set when coroutine has run enough to perform its
responsibilities.
Returns:
*task* in its final state.
An asyncio.Future based on this coroutine has very similar semantics to the
required *future* argument, but this is subject to change.
The coroutine is intended to facilitate progress of the task,
regardless of the rp.Task results. The provided *future* allows the rp.Task
results to be interpreted and semantically translated. rp.Task failure is
translated into an exception on *future*. The *future* has a different
exposure than the coroutine return value, as well: again, the *future* is
connected to the workflow item and user-facing interface, whereas this
coroutine is a detail of the task management. Still, these two modes of output
are subject to revision without notice.
Caller should await the *ready* event before assuming the watcher task is doing its
job.
"""
try:
ready.set()
def finished():
return task.state in (rp.states.DONE, rp.states.CANCELED, rp.states.FAILED) \
or future.done() \
or final
while not finished():
# Let the watcher wake up periodically to check for state changes.
# TODO: (#96) Use a control thread to manage *threading* primitives and
# translate to asyncio primitives.
done, pending = await asyncio.wait(
[future], timeout=0.05, return_when=asyncio.FIRST_COMPLETED)
if future.cancelled():
assert future in done
if task.state != rp.states.CANCELED:
logger.debug(
'Propagating cancellation from scalems future to rp task.'
)
task.cancel()
return task
if final:
logger.debug(f'Handling finalization for RP task {task.uid}')
if final.failed.is_set():
if not future.cancelled():
assert not future.done()
assert future in pending
logger.debug('Propagating RP Task failure.')
# TODO: Provide more useful error feedback.
future.set_exception(
RPTaskFailure(f'{task.uid} failed.', task=task))
elif final.canceled.is_set():
logger.debug(
'Propagating RP Task cancellation to scalems future.')
future.cancel()
raise asyncio.CancelledError(
"Managed RP.Task was cancelled.")
else:
assert final.done.is_set()
if not future.cancelled():
logger.debug(
'Publishing RP Task result to scalems Future.')
# TODO: Manage result type better.
result = task.as_dict()
future.set_result(result)
return task
if task.state in (rp.states.DONE, rp.states.CANCELED,
rp.states.FAILED):
if not final:
logger.debug(f'RP Task {task.uid} complete, but Event not '
'triggered. Possible race condition.')
except asyncio.CancelledError as e:
logger.debug(
'Propagating scalems manager task cancellation to scalems future and rp '
'task.')
future.cancel()
task.cancel()
raise e
