def process(self, crawler, request):
future = asyncio.Future()
ensure_future(self._fetch(request, future))
planned = Planned()
def build_response(f):
planned.set_result(f.result())
future.add_done_callback(build_response)
return planned
def process(self, crawler, request):
future = asyncio.Future()
ensure_future(self._fetch(request, future))
planned = Planned()
def build_response(f):
planned.set_result(f.result())
future.add_done_callback(build_response)
return planned
def get(self, requests):
for request in requests:
future = asyncio.Future()
ensure_future(self._fetch(request, future))
planned = Planned()
def build_response(f):
planned.set_result(f.result())
future.add_done_callback(build_response)
yield planned
def test_asyncio_engine(self):
req_resp_midlleware = RequestResponseMiddleware(
prefix_url=self.httpd.location,
request_factory=lambda x: x,
)
collect_middleware = CollectRequestResponseMiddleware()
downloader = AiohttpDownloader(
middlewares=[AiohttpAdapterMiddleware(), collect_middleware]
)
downloader.middlewares.insert(0, req_resp_midlleware)
pomp = AioPomp(
downloader=downloader,
pipelines=[],
)
class Crawler(DummyCrawler):
ENTRY_REQUESTS = '/root'
loop = asyncio.get_event_loop()
loop.run_until_complete(ensure_future(pomp.pump(Crawler())))
loop.close()
assert_set_equal(
set([r.url.replace(self.httpd.location, '')
for r in collect_middleware.requests]),
set(self.httpd.sitemap.keys())
)
def process(self, crawler, request):
return ensure_future(self._fetch(request))
def process(self, crawler, request):
return ensure_future(self._fetch(request))
