def add_to_map(self, url: str, action: Method, fut: asyncio.Task, **kwargs) -> None:
if fut.exception():
_logger.warning(f"Can't {action.value} on {url}: {fut.exception()}")
self.hook_add_to_map_error(url=url, action=action, fut=fut, **kwargs)
return
sub_name = kwargs.pop("sub_name")
if action == Method.ADD_SUB:
vs = VirtualSubscription(**kwargs)
self.real_map[url][sub_name] = vs
self.name_to_subscription[url][sub_name] = fut.result()
if action == Method.ADD_MI:
nodes = kwargs["nodes"]
vs = self.real_map[url][sub_name]
vs.subscribe_data_change(nodes, *astuple(kwargs["node_attr"]))
for node, handle in zip(nodes, fut.result()):
if isinstance(handle, ua.StatusCode):
# a StatusCode is returned, the request has failed.
vs.unsubscribe([node])
_logger.info(f"Node {node} subscription failed: {handle}")
# The node is invalid, remove it from both maps
if handle.name == "BadNodeIdUnknown":
_logger.warning(
f"WARNING: Abandoning {node} because it returned {handle} from {url}"
)
real_vs = self.ha_client.ideal_map[url][sub_name]
real_vs.unsubscribe([node])
continue
self.node_to_handle[url][node] = handle
self.hook_add_to_map(fut=fut, url=url, action=action, **kwargs)
def test_get_orders(self):
client = Client(self.loop)
self.loop.run_until_complete(client.connect())
# Place two test orders
buy_order = ibo.Order('buy', 110000, 'stp', aux_price=2)
task = Task(client.place_order(TEST_CONTRACT, buy_order))
self.loop.run_until_complete(task)
buy_order_id = task.result()
self.assertTrue(buy_order_id > 0)
sell_order = ibo.Order('sell', 110000, 'stp', aux_price=0.2)
task = Task(client.place_order(TEST_CONTRACT, sell_order))
self.loop.run_until_complete(task)
sell_order_id = task.result()
self.assertTrue(sell_order_id > 0)
# Retrieve the open orders matching our test ids
task = Task(client.get_orders())
self.loop.run_until_complete(task)
self.assertIsNotNone(task.result())
orders = [x for x in task.result()
if x.order_id in (buy_order_id, sell_order_id)]
self.assertEqual(2, len(orders))
# Cancel the orders and disconnect
task = Task(client.cancel_order(buy_order_id))
self.loop.run_until_complete(task)
task = Task(client.cancel_order(sell_order_id))
self.loop.run_until_complete(task)
self.loop.run_until_complete(client.disconnect())
def _on_done(self, task: asyncio.Task) -> None:
if not task.cancelled():
try:
task.result()
except Exception:
log.exception("Async-iterator task ended with error")
pass # TODO: log it or something...
def task_done_callback(self, task: asyncio.Task):
try:
task.result()
except asyncio.CancelledError:
pass
except Exception as error:
log.exception(f"Task failed.", exc_info=error)
def _handle_task_result(task: asyncio.Task) -> None:
try:
task.result()
except asyncio.CancelledError:
pass # Task cancellation should not be logged as an error.
except Exception: # pylint: disable=broad-except
logging.exception('Exception raised by task = %r', task)
async def _async_run_long_action(self, long_task: asyncio.Task) -> None:
"""Run a long task while monitoring for stop request."""
async def async_cancel_long_task() -> None:
# Stop long task and wait for it to finish.
long_task.cancel()
with suppress(Exception):
await long_task
# Wait for long task while monitoring for a stop request.
stop_task = self._hass.async_create_task(self._stop.wait())
try:
await asyncio.wait({long_task, stop_task},
return_when=asyncio.FIRST_COMPLETED)
# If our task is cancelled, then cancel long task, too. Note that if long task
# is cancelled otherwise the CancelledError exception will not be raised to
# here due to the call to asyncio.wait(). Rather we'll check for that below.
except asyncio.CancelledError:
await async_cancel_long_task()
raise
finally:
stop_task.cancel()
if long_task.cancelled():
raise asyncio.CancelledError
if long_task.done():
# Propagate any exceptions that occurred.
long_task.result()
else:
# Stopped before long task completed, so cancel it.
await async_cancel_long_task()
def task_done_callback(task: asyncio.Task):
"""Properly handle and log errors in the startup task."""
try:
task.result()
except asyncio.CancelledError:
pass
except Exception as error:
log.exception("Failed to initialize the cog.", exc_info=error)
def done_callback(task: asyncio.Task) -> None:
"""Send exception when consuming task have been cancelled."""
try:
task.result()
except asyncio.CancelledError:
self.log.info(
f"The consume task of {type(self).__name__} was canceled. Messages may be lost."
)
def test_get_account_name(self):
client = Client(self.loop)
self.loop.run_until_complete(client.connect())
task = Task(client.get_account_name())
self.loop.run_until_complete(task)
self.assertIsNotNone(task.result())
self.loop.run_until_complete(client.disconnect())
account_name = task.result()
self.assertTrue(len(account_name) > 0)
def _session_done_callback(self, task: asyncio.Task) -> None: try: task.result() # this is needed to raise ANY exception that occurred during the pagination process except asyncio.CancelledError: pass finally: self._is_running = False # already set in :meth:`.stop()`, but just in case this was reached without that method being called if self in ReactionMenu._active_sessions: ReactionMenu._active_sessions.remove(self)
def handle_result(task: Task) -> None:
try:
task.result()
except CancelledError:
LOGGER.warning("Task %s was cancelled.", task)
except Exception as e:
LOGGER.exception("An exception occurred while handling a future: %s", str(e))
finally:
self._background_tasks.discard(task)
self._await_tasks.discard(task)
def display_loop(term, es_servers): """Starts and waits on inputs, slightly mangles data and sends them to be rendered This corresponds loosly with an event loop in game programming """ query = "" # avoid recreating this all the time due to implicit dns lookups in creation es = Elasticsearch(es_servers) loop = asyncio.get_event_loop() input = Task(input_loop(loop, term, query)) search = Task(search_loop(loop, es, query)) jobs = [input, search] while True: done, pending = yield from asyncio.wait(jobs) jobs = [] # this needs to come before input handling as it may try # and cancel a compleated job before its processed. as resuls # take a bit to come back its better to just briefly display them if search in done: results = search.result() render_search_results(term, results) if input in done: query = input.result() render_query_field(term, query) # we wrap this is a task for the membership query above # if we just pass in a corutine, asyncio.wait takes it upon # itself to wrap that in a Task and the comparison will always # fail input = Task(input_loop(loop, term, query)) jobs.append(input) # ES does not return results to us for short queries, so just short # circuit them and avoid network traffic if len(query) > 1: # we wont be using the results anymore so just discard the worker search.cancel() search = Task(search_loop(loop, es, query)) jobs.append(search) else: # force a clear of the results render_search_results(term, []) sys.stdout.flush() # Make sure we resubmit any jobs that have not yet been compleated # otherwise we leak them and things go into limbo jobs += pending
def _handle_task_result(task: asyncio.Task, room=1, player=0, writer=None):
try:
task.result()
except asyncio.CancelledError:
pass # Task cancellation should not be logged as an error.
except ConnectionError:
writer.close()
my_logger.exception(
f"Connection to room {room}: player {player} is lost")
except Exception: # pylint: disable=broad-except
my_logger.exception('Exception raised by task = %r', task)
def parsePkResult(results: asyncio.Task,
vsInfoTask: asyncio.Task) -> List[str]:
'''
取出异步任务结果
并将结果解析拼接成消息字符串
'''
orders: List[str] = results.result()
vsInfo: str = vsInfoTask.result()
# split = "----------------"
# return [f"{order}\n{split}\n对家详情:\n{vsInfo}" for order in orders]
return [templates.pkTemplate(order, vsInfo) for order in orders]
def _on_done(self, task: asyncio.Task) -> None:
if task in self._tasks:
# Probably task was removed from _cleanup
self._tasks.remove(task)
self._done -= 1
if self._done <= 0:
self._queue.close()
if not task.cancelled():
try:
task.result()
except Exception:
pass # TODO: log exception
def _handle_task_result(
task: asyncio.Task,
logger: logging.Logger,
exception_message: str,
exception_message_args: Tuple[Any, ...] = (),
) -> None:
try:
task.result()
except asyncio.CancelledError:
pass
except Exception:
logger.exception(exception_message, *exception_message_args)
def _reraise_if_necessary(task: asyncio.Task) -> None:
try:
if not task.cancelled():
task.result()
except Exception as error: # pylint: disable=broad-except
self.logger.error(
f"Exiting from servo main loop do to error: {error} (task={task})"
)
self.logger.opt(exception=error).trace(
f"Exception raised by task {task}"
)
raise error # Ensure that we surface the error for handling
def _handle_task_result(
task: asyncio.Task,
*,
logger: logging.Logger,
message: str,
message_args: Tuple[Any, ...] = (),
) -> None:
try:
task.result()
except asyncio.CancelledError:
pass # Task cancellation should not be logged as an error.
# Ad the pylint ignore: we want to handle all exceptions here so that the result of the task
# is properly logged. There is no point re-raising the exception in this callback.
except Exception: # pylint: disable=broad-except
logger.exception(message, *message_args)
def _asyncio_exception_callback(self, task: asyncio.Task):
"""Used for "handling" unhandled exceptions in :meth:`ReactionMenu._execute_session`. Because that method is running in the background (asyncio.create_task), a callback is needed
when an exception is raised or else all exceptions are suppressed/lost until the program terminates, which it won't because it's a bot. This re-raises those exceptions
if any so proper debugging can occur both on my end and the users end (using :attr:`ButtonType.CALLER`)
.. added:: v1.0.3
.. changes::
v1.0.5
Added try/except to properly handle/display the appropriate tracebacks for when tasks are cancelled
"""
try:
task.result()
except asyncio.CancelledError:
pass
def callback(self, task: Task):
if self._event.is_set():
return
# Check if the code is valid
try:
response = task.result()
code = response.status
for valid_range in self.valid_codes:
if ((len(valid_range) == 2
and valid_range[0] <= code <= valid_range[1])
or valid_range[0] == code):
break
# Spawn new task if code is invalid
else:
self.spawn()
return
# Spawn new task if TimeoutError was recieved
except TimeoutError as _:
self.spawn()
return
for _task in self._tasks:
_task.cancel()
self._result = response.read()
self._event.set()
def url_response_callback(response_task: Task, url: str) -> None:
try:
response: Response = response_task.result()
response.raise_for_status()
except (TimeoutError, HTTPStatusError) as e:
LOG.warning(e)
return
except:
LOG.exception('Unexpected error.')
return
if response.headers.get('content-type').startswith('text/html'):
if not parse_html:
return
html_parse_result: HTMLScriptParseResult = html_content_to_parse_result(
html_content=response.text)
for script_num, script_content in enumerate(
html_parse_result.script_contents, start=1):
path_to_endpoint_candidate_matches[
f'{url} script #{script_num}'] = find_endpoint_candidates(
content=script_content, color_context=color_context)
external_resources_urls.update(
script_source if bool(urlparse(script_source).netloc
) else urljoin(url, script_source)
for script_source in html_parse_result.script_sources)
else:
path_to_endpoint_candidate_matches[url] = find_endpoint_candidates(
content=response.text, color_context=color_context)
def test_get_contract(self):
client = Client(self.loop)
self.loop.run_until_complete(client.connect())
task = Task(client.get_contract(TEST_CONTRACT))
self.loop.run_until_complete(task)
self.assertIsNotNone(task.result())
self.loop.run_until_complete(client.disconnect())
def _on_completed(task: asyncio.Task):
# 任务重启
def _crawl_job_retry(crawl_job: CrawlJob):
if crawl_job.retry_count < self.max_retry_count: # 请求失败 且重试次数小于指定值 重新入队 等待下次调度
crawl_job.retry_count += 1
self.append_job(crawl_job)
else: # 请求失败 耗尽重试次数
self.event_crawl_job_finish.add_page_fail_count()
crawl_job: CrawlJob = task2job[task]
html_content:str = task.result()
if html_content != "": # 正确获取 url 内容 调用回调
count_added_proxy = 0
try:
count_added_proxy = crawl_job.callback(html_content)
except Exception as e: # 回调异常 重启任务
_crawl_job_retry(crawl_job)
logging.error("Crawl Job 回调异常: %s traceback.format_exc():____%s" % (e, traceback.format_exc()))
# 将错误内容写至日志目录中
with open("./production/log/crawl_exception_{}.html".format(time.time()), "w") as f:
f.write(html_content)
else: # 回调成功 成功向代理池中添加代理
self.event_crawl_job_finish.add_page_count()
self.event_crawl_job_finish.add_proxy_count(count_added_proxy)
logging.info("完成抓取任务 {} 添加代理 {} 个".format(crawl_job.target_url, count_added_proxy))
else: # 获取 html content 为空 重启任务
_crawl_job_retry(crawl_job)
if task in task2job: del task2job[task]
else: logging.error("task not int task2job")
# 触发 crawl job 全部完成事件
if len(task2job) == 0 and self.crawl_job_queue.qsize() == 0:
self.event_crawl_job_finish.set()
def _handle_done_tasks(task: asyncio.Task) -> Any:
try:
return task.result()
except asyncio.CancelledError:
pass # Task cancellation should not be logged as an error.
except Exception as err:
logging.exception(f"Exception raised by Task {task}, error: {err}")
def _fire_or_add_pending_done_callbacks(
self, interceptors_task: asyncio.Task) -> None:
if not self._pending_add_done_callbacks:
return
call_completed = False
try:
call = interceptors_task.result()
if call.done():
call_completed = True
except (AioRpcError, asyncio.CancelledError):
call_completed = True
if call_completed:
for callback in self._pending_add_done_callbacks:
callback(self)
else:
for callback in self._pending_add_done_callbacks:
callback = functools.partial(self._wrap_add_done_callback,
callback)
call.add_done_callback(callback)
self._pending_add_done_callbacks = []
def result_callback(response_task: Task, url: str) -> None:
LOG.debug(ProgressStatus(iteration=download_summary.num_completed, total=len(urls)))
try:
response: Response = response_task.result()
response.raise_for_status()
except TimeoutException:
LOG.warning(f'Timed out: {url}')
download_summary.num_timeout += 1
return
except HTTPStatusError as e:
LOG.warning(e.args[0].splitlines()[0])
download_summary.num_status_errors += 1
return
except:
LOG.exception(f'Unexpected error: {url}')
download_summary.num_unexpected_error += 1
return
download_summary.num_downloaded += 1
if use_hashing:
download_path: Path = output_directory / Path(sha256(response.content).hexdigest())
else:
download_path: Path = output_directory / PurePath(urlparse(url=url).path).name
if download_path.exists():
LOG.warning(f'File already exists at download path: {download_path}')
download_summary.num_duplicates += 1
else:
download_path.write_bytes(response.content)
def retrieve_exception_cb(task: asyncio.Task):
""" retrieve potential result/exception from orphan tasks """
try:
result = task.result()
self.logger.debug(f'orphan task {task!r} done with {result}')
except:
self.logger.debug(f'orphan task {task!r} got exception',
exc_info=True)
def ui_closed(self, task: asyncio.Task) -> None:
if task.exception():
self.logger.error("UI failed with an exception: %s",
task.exception())
self.returncode = 1
else:
self.returncode = task.result()
self.stop_event.set()
def _done(_task: asyncio.Task):
_host = _task.get_name()
_res = _task.result()
if not len(_res):
print(f"IPF device not found: {_host}")
return
callback(_host, _res[0])
def callback(task: Task):
result = task.result()
if isinstance(result.response, Exception):
self.synchronize_resource(
result.resource, result.response, fun_name, False, False)
else:
self.synchronize_resource(
result.resource, result.response, fun_name, True, True)
def is_done(task: asyncio.Task):
if task.done():
try:
result = task.result()
print(result.content)
except:
pass
return True
return False
def _handle_response_ready(self, task: asyncio.Task) -> None:
"""Handle delayed response."""
response = self.response
self.response = None
try:
response.body = task.result()
except Exception: # pylint: disable=broad-except
response = self.handler.generic_failure_response()
self.send_response(response)
def test_get_account(self):
# This can be a time-consuming test (sometimes IB takes a minute or so
# to return all of the account data)
client = Client(self.loop)
self.loop.run_until_complete(client.connect())
task = Task(client.get_account())
self.loop.run_until_complete(task)
self.assertIsNotNone(task.result())
self.loop.run_until_complete(client.disconnect())
def _predict_callback(self, batched: BatchedRequests, predict_task: Task):
try:
batched_response = predict_task.result()
responses = batched.split_response(batched_response)
for internal_id, response in responses.items():
self._async_responses[internal_id].set_result(response)
except Exception as err:
for internal_id in batched.inference_requests.keys():
self._async_responses[internal_id].set_exception(err)
async def _fix_task(self, task: asyncio.Task) -> None:
# Insure we "retrieve" the result of failed tasks
exc = task.exception()
if exc is None:
task.result()
fixer = self._tasks[task]
if fixer is None:
raise RuntimeError(
f"{task} finished and there is no fixer!") from exc
new_task = fixer(task)
if not isinstance(new_task, asyncio.Task) and isawaitable(new_task):
new_task = await new_task
if isinstance(new_task, asyncio.Task):
del self._tasks[task]
self._tasks[new_task] = fixer
else:
raise TypeError(f"{fixer}(task) failed to return a task, returned:"
f"{new_task}!") from exc
async def follow_through(task: Task,
loop: Optional[AbstractEventLoop]=None) -> Any:
"""Wait for a task to complete (even if canceled while waiting).
**Note**: this function is a coroutine.
Not propagating cancellation to a child task and returning without waiting
for the child task to complete is a common cause of "event loop closed"
``RuntimeError`` exceptions, especially during program shutdown.
Therefore, this becomes a common pattern:
.. code-block:: python
try:
await asyncio.wait({task})
except asyncio.CancelledError:
task.cancel()
await asyncio.wait({task})
raise
return task.result()
This is not trivial and must be done so many times in a program that spawns
child tasks that it merits a helper method.
:param task: The ``asyncio.Task`` object to see through to completion.
:param loop: The event loop to use for awaiting. Defaults to the current
event loop.
.. versionadded:: 0.3
"""
loop = loop or asyncio.get_event_loop()
try:
await asyncio.wait({task}, loop=loop)
except asyncio.CancelledError:
await cancel(task, loop=loop)
raise
return task.result()
