示例#1
0
文件： spider.py 项目： canozden/slybot 
 def _get_item_requests(self, templates):
     reqs = []
     urls = [x.url for x in templates]
     for scheme, netloc in iter_unique_scheme_netloc(urls):
         r = Request("%s://%s/" % (scheme, netloc), callback=self.parse, dont_filter=True)
         reqs.append(r)
     return reqs
示例#2
0
 def _get_allowed_domains(self, templates):
     urls = [x.url for x in templates]
     urls += self.start_urls
     return [x[1] for x in iter_unique_scheme_netloc(urls)]