def _get_allowed_domains(self, spec):
urls = [x['url'] for x in spec['templates']]
urls += [
x['url'] for x in spec.get('init_requests', [])
if x['type'] == 'start'
]
urls += self._start_urls.allowed_domains
return [domain for scheme, domain in iter_unique_scheme_hostname(urls)]
def _get_allowed_domains(self, templates):
urls = [x['url'] for x in templates]
urls += [x.url for x in self._start_requests]
return [x[1] for x in iter_unique_scheme_hostname(urls)]
def _get_allowed_domains(self, templates):
urls = [x['url'] for x in templates]
urls += [x.url for x in self._start_requests]
return [x[1] for x in iter_unique_scheme_hostname(urls)]
def _get_allowed_domains(self, templates):
urls = [x.url for x in templates]
urls += self.start_urls
return [x[1] for x in iter_unique_scheme_hostname(urls)]
def _get_allowed_domains(self, templates):
urls = [x.url for x in templates]
urls += self.start_urls
return [x[1] for x in iter_unique_scheme_hostname(urls)]
def _get_allowed_domains(self, spec):
urls = [x['url'] for x in spec['templates']]
urls += [x['url'] for x in spec.get('init_requests', [])
if x['type'] == 'start']
urls += self._start_urls.allowed_domains
return [domain for scheme, domain in iter_unique_scheme_hostname(urls)]
