def add_to_map(self, url: str, action: Method, fut: asyncio.Task, **kwargs) -> None: if fut.exception(): _logger.warning(f"Can't {action.value} on {url}: {fut.exception()}") self.hook_add_to_map_error(url=url, action=action, fut=fut, **kwargs) return sub_name = kwargs.pop("sub_name") if action == Method.ADD_SUB: vs = VirtualSubscription(**kwargs) self.real_map[url][sub_name] = vs self.name_to_subscription[url][sub_name] = fut.result() if action == Method.ADD_MI: nodes = kwargs["nodes"] vs = self.real_map[url][sub_name] vs.subscribe_data_change(nodes, *astuple(kwargs["node_attr"])) for node, handle in zip(nodes, fut.result()): if isinstance(handle, ua.StatusCode): # a StatusCode is returned, the request has failed. vs.unsubscribe([node]) _logger.info(f"Node {node} subscription failed: {handle}") # The node is invalid, remove it from both maps if handle.name == "BadNodeIdUnknown": _logger.warning( f"WARNING: Abandoning {node} because it returned {handle} from {url}" ) real_vs = self.ha_client.ideal_map[url][sub_name] real_vs.unsubscribe([node]) continue self.node_to_handle[url][node] = handle self.hook_add_to_map(fut=fut, url=url, action=action, **kwargs)
def test_get_orders(self): client = Client(self.loop) self.loop.run_until_complete(client.connect()) # Place two test orders buy_order = ibo.Order('buy', 110000, 'stp', aux_price=2) task = Task(client.place_order(TEST_CONTRACT, buy_order)) self.loop.run_until_complete(task) buy_order_id = task.result() self.assertTrue(buy_order_id > 0) sell_order = ibo.Order('sell', 110000, 'stp', aux_price=0.2) task = Task(client.place_order(TEST_CONTRACT, sell_order)) self.loop.run_until_complete(task) sell_order_id = task.result() self.assertTrue(sell_order_id > 0) # Retrieve the open orders matching our test ids task = Task(client.get_orders()) self.loop.run_until_complete(task) self.assertIsNotNone(task.result()) orders = [x for x in task.result() if x.order_id in (buy_order_id, sell_order_id)] self.assertEqual(2, len(orders)) # Cancel the orders and disconnect task = Task(client.cancel_order(buy_order_id)) self.loop.run_until_complete(task) task = Task(client.cancel_order(sell_order_id)) self.loop.run_until_complete(task) self.loop.run_until_complete(client.disconnect())
def _on_done(self, task: asyncio.Task) -> None: if not task.cancelled(): try: task.result() except Exception: log.exception("Async-iterator task ended with error") pass # TODO: log it or something...
def task_done_callback(self, task: asyncio.Task): try: task.result() except asyncio.CancelledError: pass except Exception as error: log.exception(f"Task failed.", exc_info=error)
def _handle_task_result(task: asyncio.Task) -> None: try: task.result() except asyncio.CancelledError: pass # Task cancellation should not be logged as an error. except Exception: # pylint: disable=broad-except logging.exception('Exception raised by task = %r', task)
async def _async_run_long_action(self, long_task: asyncio.Task) -> None: """Run a long task while monitoring for stop request.""" async def async_cancel_long_task() -> None: # Stop long task and wait for it to finish. long_task.cancel() with suppress(Exception): await long_task # Wait for long task while monitoring for a stop request. stop_task = self._hass.async_create_task(self._stop.wait()) try: await asyncio.wait({long_task, stop_task}, return_when=asyncio.FIRST_COMPLETED) # If our task is cancelled, then cancel long task, too. Note that if long task # is cancelled otherwise the CancelledError exception will not be raised to # here due to the call to asyncio.wait(). Rather we'll check for that below. except asyncio.CancelledError: await async_cancel_long_task() raise finally: stop_task.cancel() if long_task.cancelled(): raise asyncio.CancelledError if long_task.done(): # Propagate any exceptions that occurred. long_task.result() else: # Stopped before long task completed, so cancel it. await async_cancel_long_task()
def task_done_callback(task: asyncio.Task): """Properly handle and log errors in the startup task.""" try: task.result() except asyncio.CancelledError: pass except Exception as error: log.exception("Failed to initialize the cog.", exc_info=error)
def done_callback(task: asyncio.Task) -> None: """Send exception when consuming task have been cancelled.""" try: task.result() except asyncio.CancelledError: self.log.info( f"The consume task of {type(self).__name__} was canceled. Messages may be lost." )
def test_get_account_name(self): client = Client(self.loop) self.loop.run_until_complete(client.connect()) task = Task(client.get_account_name()) self.loop.run_until_complete(task) self.assertIsNotNone(task.result()) self.loop.run_until_complete(client.disconnect()) account_name = task.result() self.assertTrue(len(account_name) > 0)
def _session_done_callback(self, task: asyncio.Task) -> None: try: task.result() # this is needed to raise ANY exception that occurred during the pagination process except asyncio.CancelledError: pass finally: self._is_running = False # already set in :meth:`.stop()`, but just in case this was reached without that method being called if self in ReactionMenu._active_sessions: ReactionMenu._active_sessions.remove(self)
def handle_result(task: Task) -> None: try: task.result() except CancelledError: LOGGER.warning("Task %s was cancelled.", task) except Exception as e: LOGGER.exception("An exception occurred while handling a future: %s", str(e)) finally: self._background_tasks.discard(task) self._await_tasks.discard(task)
def display_loop(term, es_servers): """Starts and waits on inputs, slightly mangles data and sends them to be rendered This corresponds loosly with an event loop in game programming """ query = "" # avoid recreating this all the time due to implicit dns lookups in creation es = Elasticsearch(es_servers) loop = asyncio.get_event_loop() input = Task(input_loop(loop, term, query)) search = Task(search_loop(loop, es, query)) jobs = [input, search] while True: done, pending = yield from asyncio.wait(jobs) jobs = [] # this needs to come before input handling as it may try # and cancel a compleated job before its processed. as resuls # take a bit to come back its better to just briefly display them if search in done: results = search.result() render_search_results(term, results) if input in done: query = input.result() render_query_field(term, query) # we wrap this is a task for the membership query above # if we just pass in a corutine, asyncio.wait takes it upon # itself to wrap that in a Task and the comparison will always # fail input = Task(input_loop(loop, term, query)) jobs.append(input) # ES does not return results to us for short queries, so just short # circuit them and avoid network traffic if len(query) > 1: # we wont be using the results anymore so just discard the worker search.cancel() search = Task(search_loop(loop, es, query)) jobs.append(search) else: # force a clear of the results render_search_results(term, []) sys.stdout.flush() # Make sure we resubmit any jobs that have not yet been compleated # otherwise we leak them and things go into limbo jobs += pending
def _handle_task_result(task: asyncio.Task, room=1, player=0, writer=None): try: task.result() except asyncio.CancelledError: pass # Task cancellation should not be logged as an error. except ConnectionError: writer.close() my_logger.exception( f"Connection to room {room}: player {player} is lost") except Exception: # pylint: disable=broad-except my_logger.exception('Exception raised by task = %r', task)
def parsePkResult(results: asyncio.Task, vsInfoTask: asyncio.Task) -> List[str]: ''' 取出异步任务结果 并将结果解析拼接成消息字符串 ''' orders: List[str] = results.result() vsInfo: str = vsInfoTask.result() # split = "----------------" # return [f"{order}\n{split}\n对家详情:\n{vsInfo}" for order in orders] return [templates.pkTemplate(order, vsInfo) for order in orders]
def _on_done(self, task: asyncio.Task) -> None: if task in self._tasks: # Probably task was removed from _cleanup self._tasks.remove(task) self._done -= 1 if self._done <= 0: self._queue.close() if not task.cancelled(): try: task.result() except Exception: pass # TODO: log exception
def _handle_task_result( task: asyncio.Task, logger: logging.Logger, exception_message: str, exception_message_args: Tuple[Any, ...] = (), ) -> None: try: task.result() except asyncio.CancelledError: pass except Exception: logger.exception(exception_message, *exception_message_args)
def _reraise_if_necessary(task: asyncio.Task) -> None: try: if not task.cancelled(): task.result() except Exception as error: # pylint: disable=broad-except self.logger.error( f"Exiting from servo main loop do to error: {error} (task={task})" ) self.logger.opt(exception=error).trace( f"Exception raised by task {task}" ) raise error # Ensure that we surface the error for handling
def _handle_task_result( task: asyncio.Task, *, logger: logging.Logger, message: str, message_args: Tuple[Any, ...] = (), ) -> None: try: task.result() except asyncio.CancelledError: pass # Task cancellation should not be logged as an error. # Ad the pylint ignore: we want to handle all exceptions here so that the result of the task # is properly logged. There is no point re-raising the exception in this callback. except Exception: # pylint: disable=broad-except logger.exception(message, *message_args)
def _asyncio_exception_callback(self, task: asyncio.Task): """Used for "handling" unhandled exceptions in :meth:`ReactionMenu._execute_session`. Because that method is running in the background (asyncio.create_task), a callback is needed when an exception is raised or else all exceptions are suppressed/lost until the program terminates, which it won't because it's a bot. This re-raises those exceptions if any so proper debugging can occur both on my end and the users end (using :attr:`ButtonType.CALLER`) .. added:: v1.0.3 .. changes:: v1.0.5 Added try/except to properly handle/display the appropriate tracebacks for when tasks are cancelled """ try: task.result() except asyncio.CancelledError: pass
def callback(self, task: Task): if self._event.is_set(): return # Check if the code is valid try: response = task.result() code = response.status for valid_range in self.valid_codes: if ((len(valid_range) == 2 and valid_range[0] <= code <= valid_range[1]) or valid_range[0] == code): break # Spawn new task if code is invalid else: self.spawn() return # Spawn new task if TimeoutError was recieved except TimeoutError as _: self.spawn() return for _task in self._tasks: _task.cancel() self._result = response.read() self._event.set()
def url_response_callback(response_task: Task, url: str) -> None: try: response: Response = response_task.result() response.raise_for_status() except (TimeoutError, HTTPStatusError) as e: LOG.warning(e) return except: LOG.exception('Unexpected error.') return if response.headers.get('content-type').startswith('text/html'): if not parse_html: return html_parse_result: HTMLScriptParseResult = html_content_to_parse_result( html_content=response.text) for script_num, script_content in enumerate( html_parse_result.script_contents, start=1): path_to_endpoint_candidate_matches[ f'{url} script #{script_num}'] = find_endpoint_candidates( content=script_content, color_context=color_context) external_resources_urls.update( script_source if bool(urlparse(script_source).netloc ) else urljoin(url, script_source) for script_source in html_parse_result.script_sources) else: path_to_endpoint_candidate_matches[url] = find_endpoint_candidates( content=response.text, color_context=color_context)
def test_get_contract(self): client = Client(self.loop) self.loop.run_until_complete(client.connect()) task = Task(client.get_contract(TEST_CONTRACT)) self.loop.run_until_complete(task) self.assertIsNotNone(task.result()) self.loop.run_until_complete(client.disconnect())
def _on_completed(task: asyncio.Task): # 任务重启 def _crawl_job_retry(crawl_job: CrawlJob): if crawl_job.retry_count < self.max_retry_count: # 请求失败 且重试次数小于指定值 重新入队 等待下次调度 crawl_job.retry_count += 1 self.append_job(crawl_job) else: # 请求失败 耗尽重试次数 self.event_crawl_job_finish.add_page_fail_count() crawl_job: CrawlJob = task2job[task] html_content:str = task.result() if html_content != "": # 正确获取 url 内容 调用回调 count_added_proxy = 0 try: count_added_proxy = crawl_job.callback(html_content) except Exception as e: # 回调异常 重启任务 _crawl_job_retry(crawl_job) logging.error("Crawl Job 回调异常: %s traceback.format_exc():____%s" % (e, traceback.format_exc())) # 将错误内容写至日志目录中 with open("./production/log/crawl_exception_{}.html".format(time.time()), "w") as f: f.write(html_content) else: # 回调成功 成功向代理池中添加代理 self.event_crawl_job_finish.add_page_count() self.event_crawl_job_finish.add_proxy_count(count_added_proxy) logging.info("完成抓取任务 {} 添加代理 {} 个".format(crawl_job.target_url, count_added_proxy)) else: # 获取 html content 为空 重启任务 _crawl_job_retry(crawl_job) if task in task2job: del task2job[task] else: logging.error("task not int task2job") # 触发 crawl job 全部完成事件 if len(task2job) == 0 and self.crawl_job_queue.qsize() == 0: self.event_crawl_job_finish.set()
def _handle_done_tasks(task: asyncio.Task) -> Any: try: return task.result() except asyncio.CancelledError: pass # Task cancellation should not be logged as an error. except Exception as err: logging.exception(f"Exception raised by Task {task}, error: {err}")
def _fire_or_add_pending_done_callbacks( self, interceptors_task: asyncio.Task) -> None: if not self._pending_add_done_callbacks: return call_completed = False try: call = interceptors_task.result() if call.done(): call_completed = True except (AioRpcError, asyncio.CancelledError): call_completed = True if call_completed: for callback in self._pending_add_done_callbacks: callback(self) else: for callback in self._pending_add_done_callbacks: callback = functools.partial(self._wrap_add_done_callback, callback) call.add_done_callback(callback) self._pending_add_done_callbacks = []
def result_callback(response_task: Task, url: str) -> None: LOG.debug(ProgressStatus(iteration=download_summary.num_completed, total=len(urls))) try: response: Response = response_task.result() response.raise_for_status() except TimeoutException: LOG.warning(f'Timed out: {url}') download_summary.num_timeout += 1 return except HTTPStatusError as e: LOG.warning(e.args[0].splitlines()[0]) download_summary.num_status_errors += 1 return except: LOG.exception(f'Unexpected error: {url}') download_summary.num_unexpected_error += 1 return download_summary.num_downloaded += 1 if use_hashing: download_path: Path = output_directory / Path(sha256(response.content).hexdigest()) else: download_path: Path = output_directory / PurePath(urlparse(url=url).path).name if download_path.exists(): LOG.warning(f'File already exists at download path: {download_path}') download_summary.num_duplicates += 1 else: download_path.write_bytes(response.content)
def retrieve_exception_cb(task: asyncio.Task): """ retrieve potential result/exception from orphan tasks """ try: result = task.result() self.logger.debug(f'orphan task {task!r} done with {result}') except: self.logger.debug(f'orphan task {task!r} got exception', exc_info=True)
def ui_closed(self, task: asyncio.Task) -> None: if task.exception(): self.logger.error("UI failed with an exception: %s", task.exception()) self.returncode = 1 else: self.returncode = task.result() self.stop_event.set()
def _done(_task: asyncio.Task): _host = _task.get_name() _res = _task.result() if not len(_res): print(f"IPF device not found: {_host}") return callback(_host, _res[0])
def callback(task: Task): result = task.result() if isinstance(result.response, Exception): self.synchronize_resource( result.resource, result.response, fun_name, False, False) else: self.synchronize_resource( result.resource, result.response, fun_name, True, True)
def is_done(task: asyncio.Task): if task.done(): try: result = task.result() print(result.content) except: pass return True return False
def _handle_response_ready(self, task: asyncio.Task) -> None: """Handle delayed response.""" response = self.response self.response = None try: response.body = task.result() except Exception: # pylint: disable=broad-except response = self.handler.generic_failure_response() self.send_response(response)
def test_get_account(self): # This can be a time-consuming test (sometimes IB takes a minute or so # to return all of the account data) client = Client(self.loop) self.loop.run_until_complete(client.connect()) task = Task(client.get_account()) self.loop.run_until_complete(task) self.assertIsNotNone(task.result()) self.loop.run_until_complete(client.disconnect())
def _predict_callback(self, batched: BatchedRequests, predict_task: Task): try: batched_response = predict_task.result() responses = batched.split_response(batched_response) for internal_id, response in responses.items(): self._async_responses[internal_id].set_result(response) except Exception as err: for internal_id in batched.inference_requests.keys(): self._async_responses[internal_id].set_exception(err)
async def _fix_task(self, task: asyncio.Task) -> None: # Insure we "retrieve" the result of failed tasks exc = task.exception() if exc is None: task.result() fixer = self._tasks[task] if fixer is None: raise RuntimeError( f"{task} finished and there is no fixer!") from exc new_task = fixer(task) if not isinstance(new_task, asyncio.Task) and isawaitable(new_task): new_task = await new_task if isinstance(new_task, asyncio.Task): del self._tasks[task] self._tasks[new_task] = fixer else: raise TypeError(f"{fixer}(task) failed to return a task, returned:" f"{new_task}!") from exc
async def follow_through(task: Task, loop: Optional[AbstractEventLoop]=None) -> Any: """Wait for a task to complete (even if canceled while waiting). **Note**: this function is a coroutine. Not propagating cancellation to a child task and returning without waiting for the child task to complete is a common cause of "event loop closed" ``RuntimeError`` exceptions, especially during program shutdown. Therefore, this becomes a common pattern: .. code-block:: python try: await asyncio.wait({task}) except asyncio.CancelledError: task.cancel() await asyncio.wait({task}) raise return task.result() This is not trivial and must be done so many times in a program that spawns child tasks that it merits a helper method. :param task: The ``asyncio.Task`` object to see through to completion. :param loop: The event loop to use for awaiting. Defaults to the current event loop. .. versionadded:: 0.3 """ loop = loop or asyncio.get_event_loop() try: await asyncio.wait({task}, loop=loop) except asyncio.CancelledError: await cancel(task, loop=loop) raise return task.result()