def _get_allowed_domains(self, spec): urls = [x['url'] for x in spec['templates']] urls += [ x['url'] for x in spec.get('init_requests', []) if x['type'] == 'start' ] urls += self._start_urls.allowed_domains return [domain for scheme, domain in iter_unique_scheme_hostname(urls)]
def _get_allowed_domains(self, templates): urls = [x['url'] for x in templates] urls += [x.url for x in self._start_requests] return [x[1] for x in iter_unique_scheme_hostname(urls)]
def _get_allowed_domains(self, templates): urls = [x.url for x in templates] urls += self.start_urls return [x[1] for x in iter_unique_scheme_hostname(urls)]
def _get_allowed_domains(self, spec): urls = [x['url'] for x in spec['templates']] urls += [x['url'] for x in spec.get('init_requests', []) if x['type'] == 'start'] urls += self._start_urls.allowed_domains return [domain for scheme, domain in iter_unique_scheme_hostname(urls)]