def extract_requests(self, response: HtmlResponse) -> Iterator[Request]:
if self.settings.getbool('AUTOPAGER'):
for url in autopager.urls(response):
with dont_increase_depth(response):
yield self._request(url, response)
for link in self.le.extract_links(response):
yield self._request(link.url, response)
def _pagination_urls(self, response):
return [
url for url in unique(
canonicalize_url(url, keep_fragments=True)
for url in autopager.urls(response))
if self.link_extractor.matches(url)
]
def _pagination_urls(self, response):
return [
url for url in
unique(
canonicalize_url(url, keep_fragments=True)
for url in autopager.urls(response)
)
if self.link_extractor.matches(url)
]
def _pagination_urls(self, response):
import autopager
return [
url for url in autopager.urls(response)
if self.link_extractor.matches(url)
]