async def main(f: asyncio.Future): await asyncio.sleep(1) try: f.set_result('I have finished') except RuntimeError as e: print(f'No longer allowed: {e}') f.cancel()
async def greet(q: asyncio.Queue, reader: asyncio.Future = None): """ loop: asyncio.AbstractEventLoop :param q: :return: """ while True: name = await q.get() if not name: print('][break][') break if name == 'q' or name == 'quit': print('][quit][') break print('salvete, {}'.format(name), flush=True) # await asyncio.sleep(0.1) print('___', flush=True) # await proc.wait() print('[terminating]') if reader: reader.cancel() print('[reader cancelled]')
async def greet(q: asyncio.Queue, reader: asyncio.Future): """ loop: asyncio.AbstractEventLoop :param q: :return: """ # proc = await asyncio.subprocess.create_subprocess_exec( # './hello.py', stdin=asyncio.subprocess.PIPE, # stdout=asyncio.subprocess.PIPE # ) proc = await asyncio.subprocess.create_subprocess_shell( './hello.py --pipe', stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE) while True: name = await q.get() print('<sending {}>'.format(name), flush=True) proc.stdin.write('{}\n'.format(name).encode()) res = await proc.stdout.read(1024) if not res: print('<break>') break print(res.decode(), flush=True) # await asyncio.sleep(0.1) await proc.wait() print('<done waiting>') # tasks = asyncio.tasks # loop.close() # print(tasks) reader.cancel() print('<reader cancelled>')
async def main2(f: asyncio.Future): await asyncio.sleep(1.0) try: f.set_result("I have finished.") except RuntimeError as e: print("No longer allowed: ", e) f.cancel()
def run_forever(self, sub_keepalive: asyncio.Future) -> None: """ Start the asyncio loop, running until it is either SIGTERM-ed or killed by keyboard interrupt. The Future parameter is used to cancel subscription Future in the case that an unexpected exception is thrown. You can also directly pass the `.subscribe()` method call instead like so: sub.run_forever(sub.subscribe(callback)) """ try: self.loop.run_forever() except (KeyboardInterrupt, concurrent.futures.CancelledError): pass finally: # 1. stop the `SubscriberClient` future, which will prevent more # tasks from being leased if not sub_keepalive.cancelled(): sub_keepalive.cancel() # 2. cancel the tasks we already have, which should just be # `worker` instances; note they have # `except CancelledError: pass` for task in asyncio.Task.all_tasks(loop=self.loop): task.cancel() # 3. stop the `asyncio` event loop self.loop.stop()
def _stop_task(key: str, future: asyncio.Future) -> None: if not future.cancelled(): future.cancel() logging.info(f"stop task:{key}") elif future.done(): logging.warning(f"task:{key} already stop") else: logging.warning(f"{key} can't stop")
async def cancel(fut: asyncio.Future) -> None: """ Cancel a future/task and await for it to cancel. This method suppresses the CancelledError """ fut.cancel() await asyncio.sleep(0) # let loop cycle with suppress(asyncio.CancelledError): await fut
class Stream: def __init__(self): self._future = Future() self._complete = Event() self._listeners = [ ] def __aiter__(self): return self async def __anext__(self): if self._complete.is_set(): raise StopAsyncIteration result = await self._future self._future = Future() return result def write(self, item, last): if self._complete.is_set(): return if last: self._set_complete() if self._future.done(): self._future = Future() self._future.set_result(item) def abort(self, exc): if self._complete.is_set(): return self._set_complete() if self._future.done(): self._future = Future() self._future.set_exception(exc) def cancel(self): self._set_complete() if self._future.done(): self._future = Future() self._future.cancel() async def completed(self): await self._complete.wait() @property def is_complete(self): return self._complete.is_set() def add_complete_listener(self, listener): self._listeners.append(listener) def _set_complete(self): self._complete.set() for listener in self._listeners: listener()
def _finalise_future(fut: Future) -> Optional[Union[Response, str]]: if fut.done(): err = fut.exception() if err: error_logger.error("%s", err, exc_info=err) else: return fut.result() else: fut.cancel()
def _done_callback(future: asyncio.Future, task: asyncio.Task): if task.cancelled(): future.cancel() return exception = task.exception() if exception is not None: future.set_exception(exception) return future.set_result(task.result())
async def wait_for_future(self, future: asyncio.Future) -> Any: done, _ = await asyncio.wait(set([future, *self._failures]), return_when=asyncio.FIRST_COMPLETED) if future not in done: future.cancel() for failure in self._failures: if failure not in done: failure.cancel() for failure in self._failures: if failure in done: raise failure.result() return future.result()
async def connect(self, disconnect_future: asyncio.Future = None): self.loop = asyncio.get_event_loop() client_logger.info("Establishing connection to %s:%d", self.address, self.port) try: self._transport, self._protocol = \ await self.loop.create_connection(lambda: ClientProtocol(self, disconnect_future), self.address, self.port) except OSError as e: client_logger.warning("Failed to establish connection:") client_logger.warning("%s", str(e)) disconnect_future.cancel() raise client_logger.info("Established connection to %s:%d", self.address, self.port)
def go(): future = Future() source = rx.from_future(future) def on_next(x): success[0] = False def on_error(err): success[1] = type(err) == asyncio.CancelledError def on_completed(): success[2] = False source.subscribe(on_next, on_error, on_completed) future.cancel()
def copy_result(from_: Future, to: Future) -> None: if not from_.done(): raise ValueError('from_ must be a completed Future') if to.done(): raise ValueError('to must NOT be a completed Future') if from_.cancelled(): to.cancel() else: exception = from_.exception() if exception is not None: to.set_exception(exception) else: result = from_.result() to.set_result(result)
def reject_on(self, future: asyncio.Future, error: Error) -> None: async def future_wrapper() -> Error: await future return error result = self._loop.create_task(future_wrapper()) result.add_done_callback(lambda f: future.cancel()) self._failures.append(result)
def get(self): """Remove and return an item from the channel. If channel is empty, wait until an item is available. This method is a coroutine. """ while self.empty() and not self._close.is_set(): getter = Future(loop=self._loop) self._getters.append(getter) try: yield from getter except ChannelClosed: raise except: getter.cancel() # Just in case getter is not done yet. if not self.empty() and not getter.cancelled(): # We were woken up by put_nowait(), but can't take # the call. Wake up the next in line. self._wakeup_next(self._getters) raise return self.get_nowait()
def _done_callback(fut: Future) -> None: try: fut.result() # try raise exception except CancelledError: fut.cancel() nonlocal finished, result, _future finished += 1 if _future is None: _future = fut for task in tasks: if task.done() or task.cancelled(): continue task.cancel() if finished == len(tasks): result.set_result(_future.result())
async def wait_with_cancellation( self, runner_task: asyncio.Future) -> TestRunStatus: try: # Here we wait for the first event, which may be: # 1. Watchdog termination signal # 2. Test runner events # 2.1 Run to end # 2.2 Timeout during run waited = await asyncio.wait( [termination_event.wait(), runner_task], return_when=asyncio.FIRST_COMPLETED) if runner_task.done(): return TestRunStatus('success', runner_task.result()) if termination_event.is_set(): logger.warning('Looks like task was cancelled by user...') runner_task.cancel() return TestRunStatus('terminated') except asyncio.TimeoutError: logger.warning('Timeout reached while waiting for tests...') return TestRunStatus('timeout')
def put(self, item): """Put an item into the channel. If the channel is full, wait until a free slot is available before adding item. If the channel is closed or closing, raise ChannelClosed. This method is a coroutine. """ while self.full() and not self._close.is_set(): putter = Future(loop=self._loop) self._putters.append(putter) try: yield from putter except ChannelClosed: raise except: putter.cancel() # Just in case putter is not done yet. if not self.full() and not putter.cancelled(): # We were woken up by get_nowait(), but can't take # the call. Wake up the next in line. self._wakeup_next(self._putters) raise return self.put_nowait(item)
class ReusableFuture: def __init__(self): self.impl = Future() self.waiting = False def __await__(self): self.waiting = True try: result = yield from self.impl return result finally: self.waiting = False self.impl = Future() def cancel(self): if self.waiting: self.impl.cancel() def set_result(self, value): if self.waiting: self.impl.set_result(value)
async def _async_poll_for_reply( self, msg_id: str, cell: NotebookNode, timeout: t.Optional[int], task_poll_output_msg: asyncio.Future, task_poll_kernel_alive: asyncio.Future, ) -> t.Dict: assert self.kc is not None new_timeout: t.Optional[float] = None if timeout is not None: deadline = monotonic() + timeout new_timeout = float(timeout) while True: try: msg = await ensure_async( self.kc.shell_channel.get_msg(timeout=new_timeout)) if msg['parent_header'].get('msg_id') == msg_id: if self.record_timing: cell['metadata']['execution'][ 'shell.execute_reply'] = timestamp() try: await asyncio.wait_for(task_poll_output_msg, self.iopub_timeout) except (asyncio.TimeoutError, Empty): if self.raise_on_iopub_timeout: task_poll_kernel_alive.cancel() raise CellTimeoutError.error_from_timeout_and_cell( "Timeout waiting for IOPub output", self.iopub_timeout, cell) else: self.log.warning( "Timeout waiting for IOPub output") task_poll_kernel_alive.cancel() return msg else: if new_timeout is not None: new_timeout = max(0, deadline - monotonic()) except Empty: # received no message, check if kernel is still alive assert timeout is not None task_poll_kernel_alive.cancel() await self._async_check_alive() await self._async_handle_timeout(timeout, cell)
async def _poll_for_reply( self, msg_id: str, timeout: t.Optional[int], task_poll_output_msg: asyncio.Future, task_poll_kernel_alive: asyncio.Future, ) -> t.Dict: assert self.kc is not None new_timeout: t.Optional[float] = None if timeout is not None: deadline = monotonic() + timeout new_timeout = float(timeout) while True: try: msg = await ensure_async(self.kc.shell_channel.get_msg(timeout=new_timeout)) if msg["parent_header"].get("msg_id") == msg_id: try: await asyncio.wait_for(task_poll_output_msg, self.iopub_timeout) except (asyncio.TimeoutError, Empty): if self.raise_on_iopub_timeout: task_poll_kernel_alive.cancel() raise ExecTimeoutError("Timeout waiting for IOPub output") else: self.log.warning("Timeout waiting for IOPub output") task_poll_kernel_alive.cancel() return msg else: if new_timeout is not None: new_timeout = max(0, deadline - monotonic()) except Empty: # received no message, check if kernel is still alive assert timeout is not None task_poll_kernel_alive.cancel() await self._check_alive() await self._handle_timeout(timeout)
def maybe_cancel(fut: asyncio.Future) -> bool: """Cancel future if it is cancellable.""" if fut is not None and not fut.done(): return fut.cancel() return False
async def cancel_task(task: asyncio.Future) -> None: task.cancel() await forever
class Bot: __slots__ = ("logger", "loop", "values", "server", "main_task", "longpoll_request", "settings", "api", "handler", "logger_file") def __init__(self, settings, logger=None, handler=None, loop=asyncio.get_event_loop()): self.logger = None self.init_logger(logger) self.logger.info("Initializing bot") self.loop = loop self.values = {} self.server = "" self.longpoll_request = None self.main_task = None self.settings = settings self.logger.info("Initializing vk clients") self.api = VkController(settings, logger=self.logger) self.logger.info("Loading plugins") if handler: self.handler = handler else: self.handler = MessageHandler(self, self.api, initiate_plugins=False) self.handler.initiate_plugins() signal.signal(signal.SIGINT, lambda x, y: self.stop_bot(True)) self.logger.info("Bot succesfully initialized") def init_logger(self, logger): if not logger: logger = logging.Logger("sketal", level=logging.INFO) formatter = logging.Formatter( fmt=u'%(filename)-10s [%(asctime)s] %(levelname)-8s: %(message)s', datefmt='%y.%m.%d %H:%M:%S') file_handler = logging.FileHandler('logs.txt') file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(formatter) self.logger_file = file_handler stream_handler = logging.StreamHandler() stream_handler.setLevel(logging.INFO) stream_handler.setFormatter(formatter) logger.addHandler(file_handler) logger.addHandler(stream_handler) self.logger = logger async def init_long_polling(self, update=0): result = None retries = 10 for x in range(retries): result = await self.api(sender=self.api.target_client ).messages.getLongPollServer(use_ssl=1, lp_version=2) if result: break time.sleep(0.5) if not result: self.logger.error("Unable to connect to VK's long polling server") exit() last_ts = 0 longpoll_key = "" if 'ts' in self.values: last_ts = self.values['ts'] if 'key' in self.values: longpoll_key = self.values['key'] if update == 0: self.server = "https://" + result['server'] longpoll_key = result['key'] last_ts = result['ts'] elif update == 3: longpoll_key = result['key'] last_ts = result['ts'] elif update == 2: longpoll_key = result['key'] self.values = { 'act': 'a_check', 'key': longpoll_key, 'ts': last_ts, 'wait': 20, 'mode': 10, 'version': 2 } async def process_longpoll_event(self, new_event): if not new_event: return event_id = new_event[0] if event_id != 4: evnt = LongpollEvent(self.api, event_id, new_event) return await self.process_event(evnt) data = MessageEventData() data.msg_id = new_event[1] data.attaches = new_event[6] data.time = int(new_event[4]) try: data.user_id = int(data.attaches['from']) data.chat_id = int(new_event[3]) - 2000000000 data.is_multichat = True del data.attaches['from'] except KeyError: data.user_id = int(new_event[3]) data.is_multichat = False # https://vk.com/dev/using_longpoll_2 flags = parse_msg_flags(new_event[2]) if flags['outbox']: if not self.settings.READ_OUT: return data.is_out = True data.full_text = new_event[5].replace('<br>', '\n') if "fwd" in data.attaches: data.forwarded = MessageEventData.parse_brief_forwarded_messages_from_lp( data.attaches["fwd"]) del data.attaches["fwd"] else: data.forwarded = [] msg = Message(self.api, data) if await self.check_event(data.user_id, data.chat_id, data.attaches): msg.is_event = True await self.process_message(msg) async def longpoll_processor(self): await self.init_long_polling() session = aiohttp.ClientSession(loop=self.loop) while True: try: self.longpoll_request = session.get(self.server, params=self.values) resp = await self.longpoll_request except aiohttp.ClientOSError: session = aiohttp.ClientSession(loop=self.loop) except (asyncio.TimeoutError, aiohttp.ServerDisconnectedError): self.logger.warning( "Long polling server doesn't respond. Changing server") await self.init_long_polling() continue try: events = json.loads(await resp.text()) except ValueError: continue failed = events.get('failed') if failed: err_num = int(failed) if err_num == 1: # 1 - update timestamp self.values['ts'] = events['ts'] elif err_num in (2, 3): # 2, 3 - new data for long polling await self.init_long_polling(err_num) continue self.values['ts'] = events['ts'] for event in events['updates']: asyncio.ensure_future(self.process_longpoll_event(event)) async def callback_processor(self, request): try: data = await request.json() except (UnicodeDecodeError, json.decoder.JSONDecodeError): return web.Response(text="ok") data_type = data["type"] if data_type == "confirmation": return web.Response(text=self.settings.CONF_CODE) obj = data["object"] if "user_id" in obj: obj['user_id'] = int(obj['user_id']) if data_type == 'message_new': data = MessageEventData.from_message_body(obj) msg = Message(self.api, data) await self.process_message(msg) else: evnt = CallbackEvent(self.api, data_type, obj) await self.process_event(evnt) return web.Response(text="ok") def longpoll_run(self, custom_process=False): self.main_task = Task(self.longpoll_processor()) if custom_process: return self.main_task self.logger.info("Started to process messages") try: self.loop.run_until_complete(self.main_task) except (KeyboardInterrupt, SystemExit): self.stop() self.logger.info("Stopped to process messages") except asyncio.CancelledError: pass def callback_run(self, custom_process=False): host = getenv('IP', '0.0.0.0') port = int(getenv('PORT', 8000)) self.logger.info("Started to process messages") try: server_generator, handler, app = self.loop.run_until_complete( self.init_app(host, port, self.loop)) server = self.loop.run_until_complete(server_generator) except OSError: self.logger.error("Address already in use: " + str(host) + ":" + str(port)) return self.main_task = Future() if custom_process: return self.main_task print("======== Running on http://{}:{} ========\n" " (Press CTRL+C to quit)".format( *server.sockets[0].getsockname())) def stop_server(): server.close() if not self.loop.is_running(): return self.loop.run_until_complete(server.wait_closed()) self.loop.run_until_complete(app.shutdown()) self.loop.run_until_complete(handler.shutdown(10)) self.loop.run_until_complete(app.cleanup()) try: self.loop.run_until_complete(self.main_task) except KeyboardInterrupt: self.stop() stop_server() self.loop.close() except asyncio.CancelledError: pass finally: stop_server() self.logger.info("Stopped to process messages") async def init_app(self, host, port, loop): app = web.Application() app.router.add_post('/', self.callback_processor) handler = app.make_handler() server_generator = loop.create_server(handler, host, port) return server_generator, handler, app def stop_bot(self, full=False): try: self.main_task.cancel() except: pass if full: self.stop() self.loop.stop() self.logger.info("Attempting to turn bot off") async def process_message(self, msg): asyncio.ensure_future(self.handler.process(msg), loop=self.loop) async def check_event(self, user_id, chat_id, attaches): if chat_id != 0 and "source_act" in attaches: photo = attaches.get("attach1_type") + attaches.get( "attach1") if "attach1" in attaches else None evnt = ChatChangeEvent(self.api, user_id, chat_id, attaches.get("source_act"), int(attaches.get("source_mid", 0)), attaches.get("source_text"), attaches.get("source_old_text"), photo, int(attaches.get("from", 0))) await self.process_event(evnt) return True return False async def process_event(self, evnt): asyncio.ensure_future(self.handler.process_event(evnt), loop=self.loop) def do(self, coroutine): if asyncio.iscoroutine(coroutine): return self.loop.run_until_complete(coroutine) return False @staticmethod def silent(func): try: func() except: pass def stop(self): self.handler.stop() self.api.stop() self.silent(self.main_task.cancel) self.logger.removeHandler(self.logger_file) self.logger_file.close()
async def kill_timeout(self, future: asyncio.Future): asyncio.sleep(self._timeout) future.cancel()
async def _stop_subscription(cls, future: asyncio.Future) -> None: future.cancel() await future result = future.result() await result.aclose()
def _close_app(app: Application, fut: asyncio.Future) -> None: if not fut.done(): logger.info("Cancelling consumer from signal") fut.cancel()
async def _wait_timeout(self, future: asyncio.Future): await asyncio.sleep(self._timeout) future.cancel() _logger.info('timeout to connect server')
async def _timeout(future: asyncio.Future, time: float) -> None: await asyncio.sleep(time) future.cancel()
def maybe_cancel(fut: asyncio.Future) -> bool: if fut is not None and not fut.done(): return fut.cancel() return False
async def _rp_task_watcher( task: rp.Task, # noqa: C901 future: asyncio.Future, final: RPFinalTaskState, ready: asyncio.Event) -> rp.Task: """Manage the relationship between an RP.Task and a scalems Future. Cancel the RP.Task if this task or the scalems.Future is canceled. Publish the RP.Task result or cancel the scalems.Future if the RP.Task is done or canceled. Arguments: task: RADICAL Pilot Task, submitted by caller. future: asyncio.Future to which *task* results should be propagated. ready: output parameter, set when coroutine has run enough to perform its responsibilities. Returns: *task* in its final state. An asyncio.Future based on this coroutine has very similar semantics to the required *future* argument, but this is subject to change. The coroutine is intended to facilitate progress of the task, regardless of the rp.Task results. The provided *future* allows the rp.Task results to be interpreted and semantically translated. rp.Task failure is translated into an exception on *future*. The *future* has a different exposure than the coroutine return value, as well: again, the *future* is connected to the workflow item and user-facing interface, whereas this coroutine is a detail of the task management. Still, these two modes of output are subject to revision without notice. Caller should await the *ready* event before assuming the watcher task is doing its job. """ try: ready.set() def finished(): return task.state in (rp.states.DONE, rp.states.CANCELED, rp.states.FAILED) \ or future.done() \ or final while not finished(): # Let the watcher wake up periodically to check for state changes. # TODO: (#96) Use a control thread to manage *threading* primitives and # translate to asyncio primitives. done, pending = await asyncio.wait( [future], timeout=0.05, return_when=asyncio.FIRST_COMPLETED) if future.cancelled(): assert future in done if task.state != rp.states.CANCELED: logger.debug( 'Propagating cancellation from scalems future to rp task.' ) task.cancel() return task if final: logger.debug(f'Handling finalization for RP task {task.uid}') if final.failed.is_set(): if not future.cancelled(): assert not future.done() assert future in pending logger.debug('Propagating RP Task failure.') # TODO: Provide more useful error feedback. future.set_exception( RPTaskFailure(f'{task.uid} failed.', task=task)) elif final.canceled.is_set(): logger.debug( 'Propagating RP Task cancellation to scalems future.') future.cancel() raise asyncio.CancelledError( "Managed RP.Task was cancelled.") else: assert final.done.is_set() if not future.cancelled(): logger.debug( 'Publishing RP Task result to scalems Future.') # TODO: Manage result type better. result = task.as_dict() future.set_result(result) return task if task.state in (rp.states.DONE, rp.states.CANCELED, rp.states.FAILED): if not final: logger.debug(f'RP Task {task.uid} complete, but Event not ' 'triggered. Possible race condition.') except asyncio.CancelledError as e: logger.debug( 'Propagating scalems manager task cancellation to scalems future and rp ' 'task.') future.cancel() task.cancel() raise e