def as_future(deferred: Deferred) -> Future:
"""
Convert Deferred to Future
:param deferred:
:type deferred: Deferred
:return:
:type: Future
"""
return deferred.asFuture(get_event_loop())
async def test_on_cancellation(self, provided_cls: AutoExtractProductData):
old_handler = signal.getsignal(SIGINT)
signal.signal(SIGINT, lambda x, y: None)
try:
lock = asyncio.Lock()
await lock.acquire()
class Provider(AutoExtractProvider):
async def do_request(self, *args, agg_stats: AggStats,
**kwargs):
await lock.acquire()
def callback(item: provided_cls):
pass
injector = get_injector_for_testing({Provider: 500})
stats = injector.crawler.stats
response = get_response_for_testing(callback)
deferred = injector.build_callback_dependencies(
response.request, response)
build_callbacks_future = Deferred.asFuture(
deferred, asyncio.get_event_loop())
async def cancel_after(sleep):
await asyncio.sleep(sleep)
pid = os.getpid()
try:
os.kill(pid, SIGINT)
except KeyboardInterrupt:
# As an effect of the SIGINT killing the process might receive
# here a KeyboardInterrupt exception. This is Ok.
pass
return CancelledError()
result = await asyncio.gather(build_callbacks_future,
cancel_after(0.05),
return_exceptions=True)
assert all([isinstance(r, CancelledError) for r in result])
page_type = provided_cls.page_type
expected_stats = {
'autoextract/total/pages/count': 1,
'autoextract/total/pages/cancelled': 1,
'autoextract/total/pages/errors': 0,
f'autoextract/{page_type}/pages/count': 1,
f'autoextract/{page_type}/pages/cancelled': 1,
f'autoextract/{page_type}/pages/errors': 0,
}
assert_stats(stats, expected_stats)
finally:
signal.signal(SIGINT, old_handler)
def deferred_to_future(d: Deferred) -> Future:
"""
.. versionadded:: VERSION
Return an :class:`asyncio.Future` object that wraps *d*.
When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await
on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
callables defined as coroutines <coroutine-support>`, you can only await on
``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
allows you to wait on them::
class MySpider(Spider):
...
async def parse(self, response):
d = treq.get('https://example.com/additional')
additional_response = await deferred_to_future(d)
"""
return d.asFuture(asyncio.get_event_loop())
def as_future(d: Deferred):
return d.asFuture(asyncio.get_event_loop())
def as_future(d: Deferred) -> Callable[..., Awaitable[Any]]:
"""Convert twisted deferred to asyncio future."""
return d.asFuture(asyncio.get_event_loop())
