Python iter_unique_scheme_hostname Examples

Example #1

File: spider.py Project: monocleman1/dd

 def _get_allowed_domains(self, spec):
     urls = [x['url'] for x in spec['templates']]
     urls += [
         x['url'] for x in spec.get('init_requests', [])
         if x['type'] == 'start'
     ]
     urls += self._start_urls.allowed_domains
     return [domain for scheme, domain in iter_unique_scheme_hostname(urls)]

Example #2

File: spider.py Project: PrasannaVenkadesh/portia

 def _get_allowed_domains(self, templates):
     urls = [x['url'] for x in templates]
     urls += [x.url for x in self._start_requests]
     return [x[1] for x in iter_unique_scheme_hostname(urls)]

Example #3

File: spider.py Project: widy28/portia

 def _get_allowed_domains(self, templates):
     urls = [x['url'] for x in templates]
     urls += [x.url for x in self._start_requests]
     return [x[1] for x in iter_unique_scheme_hostname(urls)]

Example #4

 def _get_allowed_domains(self, templates):
     urls = [x.url for x in templates]
     urls += self.start_urls
     return [x[1] for x in iter_unique_scheme_hostname(urls)]

Example #5

File: spider.py Project: BigData-Tools/slybot

 def _get_allowed_domains(self, templates):
     urls = [x.url for x in templates]
     urls += self.start_urls
     return [x[1] for x in iter_unique_scheme_hostname(urls)]

Example #6

File: spider.py Project: NamiStudio/portia

 def _get_allowed_domains(self, spec):
     urls = [x['url'] for x in spec['templates']]
     urls += [x['url'] for x in spec.get('init_requests', [])
              if x['type'] == 'start']
     urls += self._start_urls.allowed_domains
     return [domain for scheme, domain in iter_unique_scheme_hostname(urls)]