async def _scan(self, url: str, parent: str):
"""
Fetches a URL HTML text and adds all links in the text to the
task queue. If URL is not available, reports it.
"""
Reporter.scan(parent, url)
try:
res = await self.session.get(url)
except aiohttp.ClientError as e:
Reporter.error(parent, url, e)
return
if res.status >= 400:
Reporter.broken(parent, url, res.status)
return
for link in await self._find_links(res):
if link not in self.visited:
self.visited.add(link)
self.q.put_nowait((link, url))
