async def _execute(self, **kwargs):
"""Calls every callback function to yield new task."""
if self.ok:
for callback in self.callbacks:
async for task in to_asyncgen(callback, self):
yield task
else:
yield None
async def _produce_tasks_from_start_requests(self):
logger.debug("Produce initial tasks...")
async for task in to_asyncgen(self.crawler.start_requests):
if isinstance(task, Request):
if not task.callbacks:
task.add_callback(self.crawler.parse)
await self.crawler.add_task(task)
elif isinstance(task, Task):
await self.crawler.add_task(task)
async def _sandbox(self, func, *args, **kwargs) -> _TaskGenerator:
"""Wrap to the async generator and catch the exceptions during work."""
try:
async for task in to_asyncgen(func, *args, **kwargs):
yield task
except Exception as e:
if "Immediately" in e.__class__.__name__:
raise e
else:
self.exceptions.append(e)
async def fetch(self):
"""Sends a request and return the response as a task."""
try:
self.page = await self.client.newPage()
if self.url_str:
resp = await self.page.goto(self.url_str)
logger.info(f"<BrowserRequest> <{resp.status}> ({resp.url})")
else:
resp = None
async for task in to_asyncgen(self.operate_page, self.page, resp):
yield task
async for task in to_asyncgen(self.page_callback, self.page, resp):
yield task
except Exception as e:
raise e
finally:
await self.client.cookies_manager.update_from_pyppeteer(self.page)
await self.page.close()
self.page = None
async def _call_func(self, position, *args, **kwargs) -> _TaskGenerator:
func = self.funcs[position]
async for task in to_asyncgen(func, *args, **kwargs):
yield task
async def _execute(self, **kwargs):
"""Wraps :meth:`fetch`"""
async for task in to_asyncgen(self.fetch):
yield task