async def retry(ctx):
if ctx['job_try'] == 1:
raise asyncio.CancelledError()
with mock.patch.object(message_from_exc, "process", process):
assert message_from_exc(exc_type, exc, tb) is info
process.assert_called_once_with(exc_type, exc, tb)
it "creates an internal server error by default":
info = MessageFromExc()(ValueError, ValueError("wat"), None)
assert info == {
"status": 500,
"error": "Internal Server Error",
"error_code": "InternalServerError",
}
it "creates a cancelled response from cancelled":
info = MessageFromExc()(asyncio.CancelledError, asyncio.CancelledError(), None)
assert info == {
"status": 500,
"error": "Request was cancelled",
"error_code": "RequestCancelled",
}
describe "AsyncCatcher":
async it "takes in the request, info and final":
request = mock.Mock(name="request")
info = mock.Mock(name="info")
final = mock.Mock(name="final")
catcher = AsyncCatcher(request, info, final=final)
assert catcher.request is request
assert catcher.info is info
assert catcher.final is final
async def _error_app(scope: dict, receive: Callable, send: Callable) -> None:
raise asyncio.CancelledError()
async def test_execute__CancelledError(self, *, runner, execution):
await self._do_execute(runner,
execution,
side_effect=asyncio.CancelledError())
async def handler(request):
raise asyncio.CancelledError()
def fct_that_raise_cancellation_error():
import asyncio # pylint: disable=reimported
raise asyncio.CancelledError("task was cancelled, but dask does not care...")
async def send_credential(self, credential: Optional[str] = None) -> dict:
if self.cancelled:
raise asyncio.CancelledError()
return await self.queue.send_to_sync(credential,
lambda: self._set_expecting(None))
async def close(self, exc=asyncio.CancelledError()):
if self.is_closed:
return
await self.loop.create_task(self.__closer(exc))
def wrapper(self, *args, **kwargs):
if self._cancelled:
raise asyncio.CancelledError(
'The conversation was cancelled before')
return f(self, *args, **kwargs)
def delayed_raise_cancelled_error():
fut.set_exception(asyncio.CancelledError())
async def some_coro():
raise asyncio.CancelledError()
def katkaise_put(self):
self._getters.append(asyncio.CancelledError())
def katkaise_get(self):
self._put(asyncio.CancelledError())
async def test_execute_actor__cancelled_running(self, *, agent):
coro = FutureMock()
coro.side_effect = asyncio.CancelledError()
await agent._execute_actor(coro, Mock(name="aref", autospec=Actor))
coro.assert_awaited()
def test_order_fill_event_processed_before_order_complete_event(self):
self.exchange.start_tracking_order(
order_id="OID1",
exchange_order_id="38065410",
trading_pair=self.trading_pair,
order_type=OrderType.LIMIT,
trade_type=TradeType.BUY,
price=Decimal("10000"),
amount=Decimal("1"),
)
order = self.exchange.in_flight_orders.get("OID1")
complete_fill = {
"fee": 10.0,
"feeRate": 0.0014,
"feeCurrency": "HBOT",
"future": None,
"id": 7828307,
"liquidity": "taker",
"market": "BTC/USDT",
"orderId": 38065410,
"tradeId": 1,
"price": 10050.0,
"side": "buy",
"size": 1,
"time": "2019-05-07T16:40:58.358438+00:00",
"type": "order"
}
fill_message = {
"channel": "fills",
"type": "update",
"data": complete_fill,
}
update_data = {
'id': 103744440814,
'clientId': 'OID1',
'market': 'BTC/USDT',
'type': 'limit',
'side': 'buy',
'price': Decimal('10050.0'),
'size': Decimal('1'),
'status': 'closed',
'filledSize': Decimal('1'),
'remainingSize': Decimal('0.0'),
'reduceOnly': False,
'liquidation': False,
'avgFillPrice': Decimal('10050.0'),
'postOnly': True,
'ioc': False,
'createdAt': '2021-12-10T15:33:57.882329+00:00'
}
update_message = {
"channel": "orders",
"type": "update",
"data": update_data,
}
mock_user_stream = AsyncMock()
# We simulate the case when the order update arrives before the order fill
mock_user_stream.get.side_effect = [update_message, fill_message, asyncio.CancelledError()]
self.exchange.user_stream_tracker._user_stream = mock_user_stream
self.test_task = asyncio.get_event_loop().create_task(self.exchange._user_stream_event_listener())
try:
self.async_run_with_timeout(self.test_task)
except asyncio.CancelledError:
pass
self.async_run_with_timeout(order.wait_until_completely_filled())
self.assertEqual(Decimal("10"), order.fee_paid)
self.assertEqual(1, len(self.order_filled_logger.event_log))
fill_event: OrderFilledEvent = self.order_filled_logger.event_log[0]
self.assertEqual(Decimal("0"), fill_event.trade_fee.percent)
self.assertEqual([TokenAmount(complete_fill["feeCurrency"], Decimal(complete_fill["fee"]))],
fill_event.trade_fee.flat_fees)
self.assertTrue(self._is_logged(
"INFO",
f"Filled {Decimal(complete_fill['size'])} out of {order.amount} of the "
f"{order.order_type_description} order {order.client_order_id}"
))
self.assertTrue(self._is_logged(
"INFO",
f"The market buy order {order.client_order_id} has completed according to user stream."
))
self.assertEqual(1, len(self.buy_order_completed_logger.event_log))
async def index() -> NoReturn:
raise asyncio.CancelledError()
async def search_task(self, site, page, search):
"""taches de recherche"""
async with site["semaphore"]:
logger.debug("%s %i - in search_task: search '%s'", self.name,
self._episode, search.data)
content = None
try:
page = await page.browser.newPage()
if self.canceled:
raise asyncio.CancelledError
await page.setViewport({"width": 1920, "height": 1080})
if self.canceled:
raise asyncio.CancelledError
await self._going_to(page, site["going_search"])
logger.debug("%s %i - in search_task: search.data = '%s'",
self.name, self._episode, search.data)
if self.canceled:
raise asyncio.CancelledError
await search.run(page)
if self.canceled:
raise asyncio.CancelledError
await self._screenshot(page)
if self.canceled:
raise asyncio.CancelledError
await site["search_button_selector"].run(page)
if self.canceled:
raise asyncio.CancelledError
# sleep(10)
if site["search_response_selector"] is not None:
await site["search_response_selector"].run(page)
if self.canceled:
raise asyncio.CancelledError
content = await page.content() # return HTML document
# print(content)
except asyncio.CancelledError:
raise
except errors.TimeoutError as error:
logger.error("%s %i - in search_task : TimeoutError : %s",
self.name, self._episode, error)
await self._close_page(page)
# logger.debug("%s %i - in search_task: send = None", self.name,
# self._episode)
#await self.url.put(None)
raise asyncio.CancelledError("TimeoutError : {}".format(error))
except errors.NetworkError as error:
logger.error("%s %i - in search_task : NetworkError : %s",
self.name, self._episode, error)
await self._close_page(page)
# logger.debug("%s %i - in search_task: send = None", self.name,
# self._episode)
#await self.url.put(None)
raise asyncio.CancelledError("NetworkError : {}".format(error))
except errors.PageError as error:
logger.error("%s %i - in search_task : PageError : %s",
self.name, self._episode, error)
await self._close_page(page)
# logger.debug("%s %i - in search_task: send = None", self.name,
# self._episode)
#await self.url.put(None)
raise asyncio.CancelledError("PageError : {}".format(error))
except Exception as error:
logger.error("%s %i - in search_task : Exception : %s",
self.name, self._episode, error)
await self._close_page(page)
# logger.debug("%s %i - in search_task: send = None", self.name,
# self._episode)
#await self.url.put(None)
raise asyncio.CancelledError("Exception : {}".format(error))
# except:
# await self._screenshot(page, "_except")
# await self.url.put(None)
# logger.debug("raise error '%s'", search.data)
# await self._close_page(page)
# raise asyncio.CancelledError
# print(content)
soup = Soup(content, features="lxml")
ahref = soup.find_all("a", href=True)
logger.debug("%s %i - in search_task: ahref = '%s'", self.name,
self._episode, clean_str(str(ahref)))
logger.info("%s %i - in search_task: search episode %s of %s",
self.name, self._episode, self.episode,
" ".join(self.filters_and))
urls = list()
for data in ahref:
if self.as_all_ellements(self.filters_and, data.get_text()):
if self.as_one_ellement(self.filters_or, data.get_text()):
#self.torrent_page_url.append(full_url(page, data["href"]))
url = full_url(page, data["href"])
logger.debug(
"%s %i - in search_task: append url = '%s'",
self.name, self._episode, url)
#await self.url.put(url)
urls.append((site, url))
await self._close_page(page)
# logger.debug("%s %i - in search_task: send = None", self.name,
# self._episode)
#await self.url.put(None)
return urls
async def test_cancel(self):
raise asyncio.CancelledError()
async def __aexit__(self, et, exc, tb):
self._exiting = True
propagate_cancelation = False
if (exc is not None and self._is_base_error(exc)
and self._base_error is None):
self._base_error = exc
if et is asyncio.CancelledError:
if self._parent_cancel_requested:
# Only if we did request task to cancel ourselves
# we mark it as no longer cancelled.
self._parent_task.__cancel_requested__ = False
else:
propagate_cancelation = True
if et is not None and not self._aborting:
# Our parent task is being cancelled:
#
# async with TaskGroup() as g:
# g.create_task(...)
# await ... # <- CancelledError
#
if et is asyncio.CancelledError:
propagate_cancelation = True
# or there's an exception in "async with":
#
# async with TaskGroup() as g:
# g.create_task(...)
# 1 / 0
#
self._abort()
# We use while-loop here because "self._on_completed_fut"
# can be cancelled multiple times if our parent task
# is being cancelled repeatedly (or even once, when
# our own cancellation is already in progress)
while self._unfinished_tasks:
if self._on_completed_fut is None:
self._on_completed_fut = self._loop.create_future()
try:
await self._on_completed_fut
except asyncio.CancelledError:
if not self._aborting:
# Our parent task is being cancelled:
#
# async def wrapper():
# async with TaskGroup() as g:
# g.create_task(foo)
#
# "wrapper" is being cancelled while "foo" is
# still running.
propagate_cancelation = True
self._abort()
self._on_completed_fut = None
assert self._unfinished_tasks == 0
self._on_completed_fut = None # no longer needed
if self._base_error is not None:
raise self._base_error
if propagate_cancelation:
# The wrapping task was cancelled; since we're done with
# closing all child tasks, just propagate the cancellation
# request now.
raise asyncio.CancelledError()
if et is not None and et is not asyncio.CancelledError:
self._errors.append(exc)
if self._errors:
# Exceptions are heavy objects that can have object
# cycles (bad for GC); let's not keep a reference to
# a bunch of them.
errors = self._errors
self._errors = None
me = TaskGroupError('unhandled errors in a TaskGroup',
errors=errors)
raise me from None
def in_asyncio(started, seen):
# This is intentionally not async
seen.flag |= 1
raise asyncio.CancelledError()
async def _download_blob(
self) -> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]:
"""
:return: download success (bool), connected protocol (BlobExchangeClientProtocol)
"""
start_time = time.perf_counter()
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
blob_hash = self.blob.blob_hash
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
try:
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address,
self.peer_port, msg.decode())
self.transport.write(msg)
if self.connection_manager:
self.connection_manager.sent_data(
f"{self.peer_address}:{self.peer_port}", len(msg))
response: BlobResponse = await asyncio.wait_for(self._response_fut,
self.peer_timeout,
loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
blob_response = response.get_blob_response()
if self.closed.is_set():
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
log.warning(msg)
raise asyncio.CancelledError(msg)
if (not blob_response or blob_response.error) and\
(not availability_response or not availability_response.available_blobs):
log.warning("%s not in availability response from %s:%i",
self.blob.blob_hash, self.peer_address,
self.peer_port)
log.warning(response.to_dict())
return self._blob_bytes_received, self.close()
elif availability_response and availability_response.available_blobs and \
availability_response.available_blobs != [self.blob.blob_hash]:
log.warning(
"blob availability response doesn't match our request from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
elif not availability_response:
log.warning(
"response from %s:%i did not include an availability response (we requested %s)",
self.peer_address, self.peer_port, blob_hash)
return self._blob_bytes_received, self.close()
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
log.warning("data rate rejected by %s:%i", self.peer_address,
self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response or blob_response.error:
log.warning("blob can't be downloaded from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
log.warning("incoming blob hash mismatch from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if self.blob.length is not None and self.blob.length != blob_response.length:
log.warning("incoming blob unexpected length from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
f" timeout in {self.peer_timeout}"
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished,
self.peer_timeout,
loop=self.loop)
log.info(
"%s at %fMB/s", msg,
round((float(self._blob_bytes_received) /
float(time.perf_counter() - start_time)) / 1000000.0,
2))
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
return self._blob_bytes_received, self
except asyncio.TimeoutError:
return self._blob_bytes_received, self.close()
except (InvalidBlobHashError, InvalidDataError):
log.warning("invalid blob from %s:%i", self.peer_address,
self.peer_port)
return self._blob_bytes_received, self.close()
async def mywait():
foo()
raise asyncio.CancelledError()
def ctrl_c():
raise asyncio.CancelledError("Mock keyboard interrupt")
async def raises():
raise asyncio.CancelledError()
async def handler(request):
resp = web.Response(text=str(request.rel_url))
resp.write_eof = mock.Mock(side_effect=asyncio.CancelledError("error"))
return resp
def on_wait_for_futures(**kwargs):
if service._wait_for_futures.call_count >= 3:
service._futures.clear()
raise asyncio.CancelledError()
def spawn(self, *args, **kwargs):
# don't complain if group is already closed.
if self._closed:
raise asyncio.CancelledError()
return super().spawn(*args, **kwargs)
async def test__wait_for_futures__CancelledError(self, *, service):
service._futures = [Mock()]
with patch('asyncio.shield', AsyncMock()) as shield:
with patch('asyncio.wait', AsyncMock()):
shield.coro.side_effect = asyncio.CancelledError()
await service._wait_for_futures()
return V()
async it "gets the transport from spawn_transport", original_message, V:
assert V.called == []
s = await V.transport.spawn(original_message, timeout=10)
assert s is V.spawned
assert V.called == [("spawn_transport", 10)]
# And it caches the result
s = await V.transport.spawn(original_message, timeout=20)
assert s is V.spawned
assert V.called == [("spawn_transport", 10)]
async it "re gets the transport was cancelled first time", original_message, V:
assert V.called == []
V.spawn_transport.side_effect = asyncio.CancelledError()
with assertRaises(asyncio.CancelledError):
await V.transport.spawn(original_message, timeout=10)
assert V.called == [("spawn_transport", 10)]
V.spawn_transport.side_effect = lambda t: V.spawned
# And it can retry
try:
s = await V.transport.spawn(original_message, timeout=20)
except asyncio.CancelledError:
assert False, "Expected it not to use old future!"
assert s is V.spawned
assert V.called == [("spawn_transport", 10), ("spawn_transport", 20)]
async def test_CancelledError_from_worker_is_ignored(make_CustomJob):
worker = AsyncMock(side_effect=asyncio.CancelledError())
job = make_CustomJob(name='foo', label='Foo', worker=worker)
job.execute()
# Does not raise CancelledError
await job._task