def _get_item_requests(self, templates):
reqs = []
urls = [x.url for x in templates]
for scheme, netloc in iter_unique_scheme_netloc(urls):
r = Request("%s://%s/" % (scheme, netloc), callback=self.parse, dont_filter=True)
reqs.append(r)
return reqs
def _get_allowed_domains(self, templates):
urls = [x.url for x in templates]
urls += self.start_urls
return [x[1] for x in iter_unique_scheme_netloc(urls)]