def __init__(self, sort_mode=None, *args, **kwargs):
from scrapy.conf import settings
settings.overrides['DEPTH_PRIORITY'] = 1
settings.overrides['SCHEDULER_DISK_QUEUE'] = 'scrapy.squeue.PickleFifoDiskQueue'
settings.overrides['SCHEDULER_MEMORY_QUEUE'] = 'scrapy.squeue.FifoMemoryQueue'
settings.overrides['CRAWLERA_ENABLED'] = True
self.quantity = kwargs.get('quantity', 1000) # default is 1000
self.proxy = 'content.crawlera.com:8010'
self.proxy_type = 'http'
#self.user_agent = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:32.0) Gecko/20100101 Firefox/32.0'
self.user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A'
self.br = BuyerReviewsBazaarApi(called_class=self)
super(HayneedleProductSpider, self).__init__(
site_name=self.allowed_domains[0],
*args,
**kwargs)
def __init__(self, *args, **kwargs):
self.br = BuyerReviewsBazaarApi(called_class=self)
super(DebenhamsProductSpider, self).__init__(*args, **kwargs)
def __init__(self, *args, **kwargs):
self.br = BuyerReviewsBazaarApi(called_class=self)
super(DockersProductsSpider, self).__init__(
site_name=self.allowed_domains[0], *args, **kwargs)
def __init__(self, *args, **kwargs):
super(ChewyProductsSpider, self).__init__(*args, **kwargs)
self.br = BuyerReviewsBazaarApi(called_class=self)
def __init__(self, *args, **kwargs):
super(PetcoProductsSpider, self).__init__(*args, **kwargs)
self.br = BuyerReviewsBazaarApi(called_class=self)
self.product_last_page = 0
def __init__(self, *args, **kwargs):
self.br = BuyerReviewsBazaarApi(called_class=self)
self.start_index = 0
super(MicrosoftStoreProductSpider, self).__init__(*args, **kwargs)