def start_requests(self):
reqs = []
if self.BRAND_GET_PARAM:
for url in self.start_urls:
parsed = urlparse.urlparse(url)
params = urlparse.parse_qs(parsed.query)
brand = params.get(self.BRAND_GET_PARAM) if not callable(self.BRAND_GET_PARAM) else [self.BRAND_GET_PARAM()]
reqs.append(Request(url, meta={'product_brand': brand[0] if brand else None}))
if self.SPECIFIC_PRODUCTS:
for products in self.SPECIFIC_PRODUCTS:
brand = products.get('brand', '')
for url in products.get('urls', ''):
reqs.append(Request(url, meta={'product_brand': brand}))
if reqs:
return reqs
else:
return BaseSpider.start_requests(self)
def start_requests(self):
"""Combine scrape and start requests."""
return itertools.chain(CallbackMixin.scrape_requests(self),
_BaseSpider.start_requests(self))
def start_requests(self):
"""Combine scrape and start requests."""
return itertools.chain(CallbackMixin.scrape_requests(self),
_BaseSpider.start_requests(self))
def start_requests(self):
"""Check arguments before starting any requests
"""
self.parse_date_args()
return BaseSpider.start_requests(self)
