class CostcoProductsSpider(BaseProductsSpider): name = "costco_products" allowed_domains = ["costco.com"] start_urls = [] SEARCH_URL = "http://www.costco.com/CatalogSearch?pageSize=96" \ "&catalogId=10701&langId=-1&storeId=10301" \ "¤tPage=1&keyword={search_term}" selenium_retries = 5 DEFAULT_CURRENCY = u'USD' REVIEW_URL = 'http://api.bazaarvoice.com/data/products.json?passkey=bai25xto36hkl5erybga10t99&apiversion=5.5&filter=id:{product_id}&stats=reviews' def __init__(self, *args, **kwargs): self.br = BuyerReviewsBazaarApi(called_class=self) super(CostcoProductsSpider, self).__init__( site_name=self.allowed_domains[0], *args, **kwargs) def _parse_single_product(self, response): return self.parse_product(response) def parse_product(self, response): prod = response.meta['product'] meta = response.meta.copy() reqs = [] meta['reqs'] = reqs # TODO since response.body is already downloaded by scrapy # may try to run it in selenium instead of downloading the page again selenium_html = self._get_page_html_selenium(response.url) # TODO might as well use that html to extract other data for x in range(self.selenium_retries - 1): if not selenium_html: selenium_html = self._get_page_html_selenium(response.url) else: break if selenium_html: price = Selector(text=selenium_html).xpath( './/*[contains(@class, "your-price")]/span[@class="value"]/text()').extract() cond_set_value(prod, 'price', Price(priceCurrency=self.DEFAULT_CURRENCY, price=price)) # not longer available no_longer_available = response.xpath( '//*[@class="server-error" and contains(text(),' '"out of stock and cannot be added to your cart at this time")]') cond_set_value(prod, 'no_longer_available', 1 if no_longer_available else 0) if not no_longer_available and response.xpath('//h1[text()="Product Not Found"]'): prod['not_found'] = True return prod model = response.xpath('//div[@id="product-tab1"]//text()').re( 'Model[\W\w\s]*') if len(model) > 0: cond_set(prod, 'model', model) if 'model' in prod: prod['model'] = re.sub(r'Model\W*', '', prod['model'].strip()) title = response.xpath('//h1[@itemprop="name"]/text()').extract() cond_set(prod, 'title', title) # Title key must be present even if it is blank cond_set_value(prod, 'title', "") tab2 = ''.join( response.xpath('//div[@id="product-tab2"]//text()').extract() ).strip() brand = '' for i in tab2.split('\n'): if 'Brand' in i.strip(): brand = i.strip() brand = re.sub(r'Brand\W*', '', brand) if brand: prod['brand'] = brand if not prod.get("brand"): brand = response.xpath( './/*[contains(text(), "Brand:")]/following-sibling::text()[1]').extract() brand = brand[0].strip() if brand else None cond_set_value(prod, 'brand', brand) des = response.xpath('//div[@id="product-tab1"]//text()').extract() des = ' '.join(i.strip() for i in des) if '[ProductDetailsESpot_Tab1]' in des.strip(): des = response.xpath("//div[@id='product-tab1']/*[position()>1]//text()").extract() des = ' '.join(i.strip() for i in des) if des.strip(): prod['description'] = des.strip() elif des: prod['description'] = des.strip() img_url = response.xpath('//img[@itemprop="image"]/@src').extract() cond_set(prod, 'image_url', img_url) cond_set_value(prod, 'locale', 'en-US') prod['url'] = response.url # Categories categorie_filters = ['home'] # Clean and filter categories names from breadcrumb categories = list(filter((lambda x: x.lower() not in categorie_filters), map((lambda x: x.strip()), response.xpath('//*[@itemprop="breadcrumb"]//a/text()').extract()))) category = categories[-1] if categories else None cond_set_value(prod, 'categories', categories) cond_set_value(prod, 'category', category) # Minimum Order Quantity try: minium_order_quantity = re.search('Minimum Order Quantity: (\d+)', response.body_as_unicode()).group(1) cond_set_value(prod, 'minimum_order_quantity', minium_order_quantity) except: pass shipping = ''.join(response.xpath( '//*[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ",' ' "abcdefghijklmnopqrstuvwxyz"), "shipping & handling:")]' ).re('[\d\.\,]+')).strip().replace(',', '') if not shipping: shipping = ''.join(response.xpath( '//*[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ",' ' "abcdefghijklmnopqrstuvwxyz"), "shipping and handling:")]' ).re('[\d\.\,]+')).strip().replace(',', '') if shipping: cond_set_value(prod, 'shipping_cost', Price(priceCurrency=self.DEFAULT_CURRENCY, price=shipping)) shipping_included = ''.join(response.xpath( '//*[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ",' ' "abcdefghijklmnopqrstuvwxyz"),"shipping & handling included")]' ).extract()).strip().replace(',', '') or \ response.xpath( '//*[@class="merchandisingText" and ' 'contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ", ' '"abcdefghijklmnopqrstuvwxyz"), "free shipping")]') or \ ''.join(response.xpath( '//p[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ",' ' "abcdefghijklmnopqrstuvwxyz"),"shipping and handling included")]' ).extract()).strip().replace(',', '') cond_set_value(prod, 'shipping_included', 1 if shipping_included or shipping == "0.00" else 0) available_store = re.search('Item may be available in your local warehouse', response.body_as_unicode()) cond_set_value(prod, 'available_store', 1 if available_store else 0) not_available_store = re.search('Not available for purchase on Costco.com', response.body_as_unicode()) cond_set_value(prod, 'available_online', 0 if not_available_store else 1) if str(prod.get('available_online', None)) == '0' and str(prod.get('available_store', None)) == '0': prod['is_out_of_stock'] = True count_review = response.xpath('//meta[contains(@itemprop, "reviewCount")]/@content').extract() product_id = re.findall(r'\.(\d+)\.', response.url) cond_set_value(prod, 'reseller_id', product_id[0] if product_id else None) if product_id and count_review: reqs.append( Request( url=self.REVIEW_URL.format(product_id=product_id[0], index=0), dont_filter=True, callback=self.parse_buyer_reviews, meta=meta )) if reqs: return self.send_next_request(reqs, response) return prod def parse_buyer_reviews(self, response): meta = response.meta.copy() product = response.meta['product'] reqs = meta.get('reqs', []) product['buyer_reviews'] = self.br.parse_buyer_reviews_products_json(response) if reqs: return self.send_next_request(reqs, response) else: return product def send_next_request(self, reqs, response): """ Helps to handle several requests """ req = reqs.pop(0) new_meta = response.meta.copy() if reqs: new_meta["reqs"] = reqs return req.replace(meta=new_meta) def _search_page_error(self, response): if not self._scrape_total_matches(response): self.log("Costco: unable to find a match", ERROR) return True return False def _scrape_total_matches(self, response): count = response.xpath( '//*[@id="secondary_content_wrapper"]/div/p/span/text()' ).re('(\d+)') count = int(count[-1]) if count else None if not count: count = response.xpath( '//*[@id="secondary_content_wrapper"]' '//span[contains(text(), "Showing results")]/text()' ).extract() count = int(count[0].split(' of ')[1].replace('.', '').strip()) if count else None if not count: count = response.css(".table-cell.results.hidden-xs.hidden-sm.hidden-md>span").re( r"Showing\s\d+-\d+\s?of\s?([\d.,]+)") count = int(count[0].replace('.', '').replace(',', '')) if count else None return count def _scrape_product_links(self, response): links = response.xpath( '//div[contains(@class,"product-list grid")]//a[contains(@class,"thumbnail")]/@href' ).extract() for link in links: yield link, SiteProductItem() def _scrape_next_results_page_link(self, response): links = response.xpath( "//*[@class='pagination']" "/ul[2]" # [1] is for the Items Per Page section which has .active. "/li[@class='active']" "/following-sibling::li[1]" # [1] is to get just the next sibling. "/a/@href" ).extract() if links: link = links[0] else: link = None return link def _get_page_html_selenium(self, url): try: display = Display(visible=False) display.start() driver = self._init_chromium() driver.set_page_load_timeout(120) driver.set_script_timeout(120) socket.setdefaulttimeout(120) driver.set_window_size(1280, 768) driver.get(url) time.sleep(5) page_html = driver.page_source driver.quit() except Exception as e: self.log('Exception while getting page html with selenium: {}'.format(e), WARNING) self.log('### Traceback: {}'.format(traceback.format_exc()), WARNING) else: return page_html def _init_chromium(self, proxy=None, proxy_type=None): # TODO use random useragent script here? # UA = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:32.0) Gecko/20100101 Firefox/32.0" chrome_flags = webdriver.DesiredCapabilities.CHROME # this is for Chrome? chrome_options = webdriver.ChromeOptions() # this is for Chromium if proxy: chrome_options.add_argument( '--proxy-server=%s' % proxy_type+'://'+proxy) # chrome_flags["chrome.switches"] = ['--user-agent=%s' % UA] # chrome_options.add_argument('--user-agent=%s' % UA) executable_path = '/usr/sbin/chromedriver' if not os.path.exists(executable_path): executable_path = '/usr/local/bin/chromedriver' # initialize webdriver driver = webdriver.Chrome(desired_capabilities=chrome_flags, chrome_options=chrome_options, executable_path=executable_path) return driver
class PetcoProductsSpider(ProductsSpider): name = 'petco_products' allowed_domains = ['petco.com'] SEARCH_URL = ("http://www.petco.com/shop/SearchDisplay?categoryId=&storeId" "=10151&catalogId=10051&langId=-1&sType=SimpleSearch&" "resultCatEntryType=2&showResultsPage=true&searchSource=Q&" "pageView=&beginIndex=0&pageSize=48&fromPageValue=search" "&searchTerm={search_term}") SEARCH_URL_2 = ("http://www.petco.com/shop/ProductListingView?searchType=" "12&filterTerm=&langId=-1&advancedSearch=&sType=Simple" "Search&resultCatEntryType=2&catalogId=10051&searchTerm=" "{search_term}&resultsPerPage=48&emsName=&facet=&category" "Id=&storeId=10151&beginIndex={begin_index}") REVIEW_URL = ("http://api.bazaarvoice.com/data/products.json?" "passkey=dpaqzblnfzrludzy2s7v27ehz&apiversion=5.5" "&filter=id:{product_id}&stats=reviews") PRICE_URL = "http://www.petco.com/shop/GetCatalogEntryDetailsByIDView" def __init__(self, *args, **kwargs): super(PetcoProductsSpider, self).__init__(*args, **kwargs) self.br = BuyerReviewsBazaarApi(called_class=self) self.product_last_page = 0 def parse_buyer_reviews(self, response): meta = response.meta.copy() product = response.meta['product'] reqs = meta.get('reqs', []) product['buyer_reviews'] = self.br.parse_buyer_reviews_products_json( response) if reqs: return self.send_next_request(reqs, response) else: return product def _total_matches_from_html(self, response): total = response.xpath( '//*[contains(@id,"searchTotalCount")]/text()').re('\d+') return int(total[0].replace(',', '')) if total else 0 def _scrape_results_per_page(self, response): return 48 def _scrape_next_results_page_link(self, response): # End of pagination if not self.product_last_page: return None begin_index = int(re.search('beginIndex=(\d+)', response.url).group(1)) num_poduct_page = self._scrape_results_per_page(response) st = response.meta['search_term'] url = self.url_formatter.format( self.SEARCH_URL_2, search_term=urllib.quote_plus(st.encode('utf-8')), begin_index=str(begin_index + num_poduct_page)) return url def _scrape_product_links(self, response): item_urls = response.xpath( '//*[@class="product-display-grid"]' '//*[@class="product-name"]/a/@href').extract() self.product_last_page = len(item_urls) for item_url in item_urls: yield item_url, SiteProductItem() def _parse_single_product(self, response): return self.parse_product(response) def _parse_title(self, response): title = response.xpath('//h1/text()').extract() return title[0].strip() if title else None def _parse_categories(self, response): categories = response.css('.breadcrumb a::text').extract() return categories if categories else None def _parse_category(self, response): categories = self._parse_categories(response) return categories[-1] if categories else None def _parse_image_url(self, response): image_url = response.xpath( '//*[@property="og:image"]/@content').extract() return image_url[0] if image_url else None def _parse_brand(self, response): brand = response.xpath('//*[@class="product-brand"]/a/text()').re( 'by.(.*)') return brand[0].strip() if brand else None def _parse_sku(self, response): sku = response.xpath("//input[@id='primarySku']/@value").extract() if len(sku[0]) < 1: sku = response.css('.product-sku::text').re(u'SKU:.(\d+)') return sku[0] if sku else None def _parse_variants(self, response): variants = [] try: variants_info = json.loads( response.xpath( '//*[contains(@id,"entitledItem_")]/text()').extract()[0]) except: variants_info = {} for attr_value in variants_info: attributes = {} variant_attribute = attr_value["Attributes"] attributes['price'] = attr_value["RepeatDeliveryPrice"]["price"] attributes['image_url'] = attr_value["ItemImage"] if variant_attribute: attr_text = attr_value["Attributes"].keys()[0].split('_') attributes[attr_text[0]] = attr_text[1] variants.append(attributes) return variants if variants else None def _parse_is_out_of_stock(self, response): status = response.xpath( '//*[@itemprop="availability" and @content="in_stock"]') return not bool(status) def _parse_shipping_included(self, response): pass def _parse_description(self, response): description = response.xpath('//*[@id="description"]').extract() return ''.join(description).strip() if description else None def send_next_request(self, reqs, response): """ Helps to handle several requests """ req = reqs.pop(0) new_meta = response.meta.copy() if reqs: new_meta["reqs"] = reqs return req.replace(meta=new_meta) def parse_product(self, response): reqs = [] product = response.meta['product'] # Set locale product['locale'] = 'en_US' # Parse title title = self._parse_title(response) cond_set_value(product, 'title', title, conv=string.strip) # Parse category category = self._parse_category(response) cond_set_value(product, 'category', category) # Parse categories categories = self._parse_categories(response) cond_set_value(product, 'categories', categories) # Parse description description = self._parse_description(response) cond_set_value(product, 'description', description) # Parse image url image_url = self._parse_image_url(response) cond_set_value(product, 'image_url', image_url) # Parse variants variants = self._parse_variants(response) cond_set_value(product, 'variants', variants) # Parse stock status out_of_stock = self._parse_is_out_of_stock(response) cond_set_value(product, 'is_out_of_stock', out_of_stock) # Sku sku = self._parse_sku(response) cond_set_value(product, 'sku', sku) # Reseller_id cond_set_value(product, 'reseller_id', sku) # Brand brand = self._parse_brand(response) cond_set_value(product, 'brand', brand) product_id = response.xpath( '//*[@id="productPartNo"]/@value').extract() if product_id: reqs.append( Request(url=self.REVIEW_URL.format(product_id=product_id[0], index=0), dont_filter=True, callback=self.parse_buyer_reviews, meta=response.meta)) price_id = response.xpath('//*[contains(@id,"entitledItem_")]/@id').re( 'entitledItem_(\d+)') cat_id = response.xpath('//script/text()').re( 'productDisplayJS.displayAttributeInfo\("(\d+)","(\d+)"') if not cat_id: cat_id = response.xpath( '//*[@name="firstAvailableSkuCatentryId_avl"]/@value').extract( ) if price_id and cat_id: text = ("storeId=10151&langId=-1&catalogId=10051&" "catalogEntryId={cat}&productId={prod_id}".format( cat=cat_id[0], prod_id=price_id[0])) reqs.append( Request(self.PRICE_URL, body=text, headers={ 'Content-Type': 'application/x-www-form-urlencoded', 'X-Requested-With': 'XMLHttpRequest' }, method='POST', meta=response.meta, callback=self._parse_price, dont_filter=True)) else: prices = map( float, response.xpath('//*[@class="product-price"]//span/text()').re( '\$([\d\.]+)')) product['price'] = Price(price=min(prices), priceCurrency="USD") if reqs: return self.send_next_request(reqs, response) return product def _parse_price(self, response): reqs = response.meta.get('reqs', []) product = response.meta['product'] raw_information = re.findall('\{.*\}', response.body, re.MULTILINE | re.DOTALL)[0] product_data = eval(raw_information) price = product_data["catalogEntry"]["offerPrice"] product['price'] = Price(price=price, priceCurrency="USD") if reqs: return self.send_next_request(reqs, response) return product
class CvsProductsSpider(BaseProductsSpider): name = 'cvs_products' allowed_domains = ["cvs.com", "api.bazaarvoice.com"] start_urls = [] SEARCH_URL = "https://www.cvs.com/search/N-0?searchTerm={search_term}" SEARCH_URL_AJAX = "https://www.cvs.com/" \ "retail/frontstore/OnlineShopService?" \ "apiKey=c9c4a7d0-0a3c-4e88-ae30-ab24d2064e43&" \ "apiSecret=4bcd4484-c9f5-4479-a5ac-9e8e2c8ad4b0&" \ "appName=CVS_WEB&" \ "channelName=WEB&" \ "contentZone=resultListZone&" \ "deviceToken=7780&" \ "deviceType=DESKTOP&" \ "lineOfBusiness=RETAIL&" \ "navNum=20&" \ "operationName=getProductResultList&" \ "pageNum={page_num}&" \ "referer={referer}&" \ "serviceCORS=False&" \ "serviceName=OnlineShopService&" \ "sortBy=relevance&" \ "version=1.0" \ REVIEW_URL = "http://api.bazaarvoice.com/data/products.json?" \ "passkey=ll0p381luv8c3ler72m8irrwo&apiversion=5.5&" \ "filter=id:{product_id}&stats=reviews" PRICE_URL = "https://www.cvs.com/retail/frontstore/productDetails?" \ "apiKey=c9c4a7d0-0a3c-4e88-ae30-ab24d2064e43&" \ "apiSecret=4bcd4484-c9f5-4479-a5ac-9e8e2c8ad4b0&" \ "appName=CVS_WEB&" \ "channelName=WEB&" \ "code={sku}&" \ "codeType=sku&" \ "deviceToken=2695&" \ "deviceType=DESKTOP&" \ "lineOfBusiness=RETAIL&" \ "operationName=getSkuPricePromotions&" \ "serviceCORS=True&" \ "serviceName=productDetails&" \ "storeId=2294&" \ "version=1.0" \ PRODUCT_DETAILS = "https://www.cvs.com/retail/frontstore/productDetails?" \ "apiKey=c9c4a7d0-0a3c-4e88-ae30-ab24d2064e43&" \ "apiSecret=4bcd4484-c9f5-4479-a5ac-9e8e2c8ad4b0&" \ "appName=CVS_WEB&" \ "channelName=WEB&" \ "code={sku}&" \ "codeType=sku&" \ "deviceToken=2695&" \ "deviceType=DESKTOP&" \ "lineOfBusiness=RETAIL&" \ "operationName=getSkuDetails&" \ "serviceCORS=True&" \ "serviceName=productDetails&" \ "version=1.0" def __init__(self, *args, **kwargs): self.br = BuyerReviewsBazaarApi(called_class=self) self.referer = None self.first_time_products = None self.current_page = 1 self.products_per_page = 20 super(CvsProductsSpider, self).__init__(site_name=self.allowed_domains[0], *args, **kwargs) settings.overrides['CRAWLERA_ENABLED'] = True def _set_brand(self, product, phrase, brands): phrase = _normalize(phrase) for brand in sorted(brands, key=len, reverse=True): if _normalize(brand) in phrase: cond_set_value(product, 'brand', brand) break def parse(self, response): print response.url if self.searchterms and not self.referer: self.referer = response.url return super(CvsProductsSpider, self).parse(response) def parse_product(self, response): brands = response.meta.get('brands', frozenset()) product = response.meta['product'] reqs = [] if 'brand' not in product: descs = response.css('.brandBanner > a ::attr(title)') if descs: desc, = descs.extract() self._set_brand(product, desc, brands) product['locale'] = "en-US" reseller_id_regex = "prodid-(\d+)" reseller_id = re.findall(reseller_id_regex, response.url) reseller_id = reseller_id[0] if reseller_id else None cond_set_value(product, 'reseller_id', reseller_id) ld_json = is_empty( response.xpath('//*[@type="application/ld+json" ' 'and contains(text(),"product")]/text()').extract()) if ld_json: try: clean_json = re.sub('([^"])\n|\t|\r', '', ld_json.replace('@', '')) product_data = json.loads(clean_json) cond_set_value(product, 'title', product_data.get('name')) cond_set_value(product, 'brand', product_data.get('brand')) ######## variants ######## variants = product_data.get('offers') if len(variants) > 1: for variant in variants: try: sku = variant['itemOffered']['sku'] price_url = self.PRICE_URL.format(sku=sku) reqs.append( Request(price_url, self._parse_variant_new, meta=response.meta)) except: pass main_variant = variants[0] description = main_variant.get( 'itemOffered', {}).get('description') or product_data.get('description') cond_set_value(product, 'description', description) main_skuID_search = re.search("skuId=(\d+)", response.url) if main_skuID_search: main_skuID = main_skuID_search.group(1) else: main_skuID = variants[0].get('itemOffered', {}).get('sku', None) cond_set_value(product, 'image_url', main_variant.get('itemOffered').get('image')) response.meta['main_skuID'] = main_skuID response.meta['offers_variants'] = variants if main_variant.get('price'): cond_set_value( product, 'price', Price(price=main_variant.get('price'), priceCurrency='USD')) # elif product_data.get('productId'): # price_url = self.PRICE_URL.format( # price_id=product_data.get('productId')) # reqs.append(Request(price_url, # self._parse_price, # meta=response.meta)) # cond_set_value(product, 'variants', # self._parse_variants(variants, main_skuID)) ############################## if main_skuID: review_url = self.REVIEW_URL.format(product_id=main_skuID) reqs.append( Request(review_url, self._parse_review, meta=response.meta)) except: import traceback print traceback.print_exc() size = response.xpath( "//form[@id='addCart']/table/tr/td[@class='col1']/" "text()[.='Size:']/../../td[2]/text()").extract() cond_set(product, 'model', size, conv=string.strip) if reqs: return self.send_next_request(reqs, response) return product def send_next_request(self, reqs, response): """ Helps to handle several requests """ req = reqs.pop(0) new_meta = response.meta.copy() if reqs: new_meta["reqs"] = reqs return req.replace(meta=new_meta) def _parse_variant_new(self, response): product = response.meta['product'] reqs = response.meta.get('reqs', []) data = json.loads(response.body) sku_price_promotions = data.get('response', {}).get('getSkuPricePromotions', []) if sku_price_promotions: sku_details = sku_price_promotions[0].get('skuDetails', []) if sku_details: variants = product.get('variants', []) variant = {} skuID = sku_details[0].get('skuId', '') variant['url'] = product.get('url', '') + "?skuId=%s" % skuID price = sku_details[0].get('priceInfo', {}).get('listPrice', None) if price: cond_set_value(product, 'price', Price(price=price, priceCurrency='USD')) variant['price'] = price main_skuID = response.meta['main_skuID'] variant['selected'] = main_skuID == skuID bohInventory = sku_details[0].get('statusInfo', {}).get('bohInventory', 0) bohStockStatus = sku_details[0].get('statusInfo', {}).get( 'bohStockStatus', 'NOTAVAILABLE') onlineOnly = sku_details[0].get('statusInfo', {}).get('onlineOnly', False) onlineStockStatus = sku_details[0].get('statusInfo', {}).get( 'onlineStockStatus', None) in_stock = False if bohInventory and bohStockStatus != 'NOTAVAILABLE': in_stock = True if onlineStockStatus == 'INSTOCK': in_stock = True variant['in_stock'] = in_stock variant['sku'] = skuID # del product['main_skuID'] variant['properties'] = {} offers_variants = response.meta['offers_variants'] for offers_variant in offers_variants: # Check that the variant is not duplicated item_offered = offers_variant.get('itemOffered', {}) this_sku = item_offered.get('sku', None) if item_offered and this_sku == skuID: attr = {} details_url = self.PRODUCT_DETAILS.format(sku=this_sku) variant['properties'] = attr reqs.append( Request(details_url, self._parse_properties, meta=response.meta)) break variants.append(variant) product['variants'] = variants if reqs: return self.send_next_request(reqs, response) return product def _parse_properties(self, response): product = response.meta['product'] reqs = response.meta.get('reqs', []) data = json.loads(response.body) getSkuDetails = data.get('response', {}).get('getSkuDetails', []) if getSkuDetails: sku_details = getSkuDetails[0].get('skuDetails', []) if len(sku_details) > 0: detail = sku_details[0]['detail'] skuSize = detail['skuSize'] weight = detail['weight'] flavor = detail['flavor'] upcNumber = detail['upcNumber'] variants = product.get('variants', []) skuID = sku_details[0].get('skuId', '') for idx, variant in enumerate(variants): # Check that the variant is not duplicated this_sku = variant.get('sku', None) if this_sku == skuID: attr = {} attr['Size'] = skuSize attr['Flavor'] = flavor attr['Weight'] = weight attr['UPCNumber'] = upcNumber variant['properties'] = attr variants[idx] = variant break product['variants'] = variants if reqs: return self.send_next_request(reqs, response) return product def _parse_variants(self, variants, main_skuID): if not variants: return None parsed_variants = [] variants_visit = set() for variant in variants: # Check that the variant is not duplicated item_offered = variant.get('itemOffered', {}) this_sku = item_offered.get('sku', None) if this_sku in variants_visit: continue variants_visit.add(this_sku) # Fill the Variant data vr = {} if variant['price']: vr['price'] = variant['price'] availability = variant.get('availability', None) vr['in_stock'] = availability == "http://schema.org/InStock" vr['selected'] = main_skuID == this_sku if item_offered: attr = {} if item_offered.get('color'): attr['Color'] = item_offered.get('color') if item_offered.get('color'): attr['Weight'] = item_offered.get('weight').get('value') vr['properties'] = attr vr['url'] = item_offered.get('url') parsed_variants.append(vr) parsed_variants[0]['selected'] = True return parsed_variants def _parse_review(self, response): product = response.meta['product'] reqs = response.meta.get('reqs', []) product['buyer_reviews'] = self.br.parse_buyer_reviews_products_json( response) if reqs: return self.send_next_request(reqs, response) return product def _scrape_total_matches(self, response): totals = response.xpath( '//*[@id="resultsTabs"]//' 'a[@title="View Products"]/text()').re('\((\d+)\)') if len(totals) > 1: self.log( "Found more than one 'total matches' for %s" % response.url, ERROR) elif totals: total = totals[0].strip() self.total_matches_int = int(total) return int(total) else: self.log("Failed to find 'total matches' for %s" % response.url, WARNING) return None def _scrape_product_links(self, response): all_links_iter = re.finditer( 'detailsLink"\s*:\s*"(.*?)(\?skuId=\d+)?",', response.body) # Clean the links for the different variants of a product links_without_dup = [] [ links_without_dup.append(item) for item in map((lambda x: x.group(1)), all_links_iter) if item not in links_without_dup ] for link in links_without_dup: yield link, SiteProductItem() def _scrape_results_per_page(self, response): return 20 def _scrape_next_results_page_link(self, response): url_parts = urlparse.urlsplit(response.url) query_string = urlparse.parse_qs(url_parts.query) ajax_search_url = self.SEARCH_URL_AJAX.format( referer=urllib.quote_plus(self.referer, ':'), page_num=self.current_page) self.current_page += 1 if self.current_page * self.products_per_page > self.total_matches_int + 30: return headers = { 'Accept': 'application/json, text/plain, */*', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'Host': 'www.cvs.com', 'Pragma': 'no-cache', 'Referer': self.referer, 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64)' ' AppleWebKit/537.36 (KHTML, like Gecko)' ' Chrome/49.0.2623.110 Safari/537.36' } return Request(ajax_search_url, self.parse, headers=headers, meta=response.meta, priority=1) def _parse_single_product(self, response): return self.parse_product(response) def _get_products(self, response): remaining = response.meta['remaining'] search_term = response.meta['search_term'] prods_per_page = response.meta.get('products_per_page') total_matches = response.meta.get('total_matches') scraped_results_per_page = response.meta.get( 'scraped_results_per_page') prods = self._scrape_product_links(response) if not prods_per_page: # Materialize prods to get its size. prods = list(prods) prods_per_page = len(prods) response.meta['products_per_page'] = prods_per_page if scraped_results_per_page is None: scraped_results_per_page = self._scrape_results_per_page(response) if scraped_results_per_page: self.log( "Found %s products at the first page" % scraped_results_per_page, INFO) else: scraped_results_per_page = prods_per_page if hasattr(self, 'is_nothing_found'): if not self.is_nothing_found(response): self.log( "Failed to scrape number of products per page", ERROR) response.meta[ 'scraped_results_per_page'] = scraped_results_per_page if total_matches is None: total_matches = self._scrape_total_matches(response) if total_matches is not None: response.meta['total_matches'] = total_matches self.log("Found %d total matches." % total_matches, INFO) else: if hasattr(self, 'is_nothing_found'): if not self.is_nothing_found(response): self.log( "Failed to parse total matches for %s" % response.url, ERROR) if total_matches and not prods_per_page: # Parsing the page failed. Give up. self.log("Failed to get products for %s" % response.url, ERROR) return for i, (prod_url, prod_item) in enumerate(islice(prods, 0, remaining)): # Initialize the product as much as possible. prod_item['site'] = self.site_name prod_item['search_term'] = search_term prod_item['total_matches'] = total_matches prod_item['results_per_page'] = prods_per_page prod_item['scraped_results_per_page'] = scraped_results_per_page # The ranking is the position in this page plus the number of # products from other pages. prod_item['ranking'] = (i + 1) + (self.quantity - remaining) if self.user_agent_key not in ["desktop", "default"]: prod_item['is_mobile_agent'] = True if prod_url is None: # The product is complete, no need for another request. yield prod_item elif isinstance(prod_url, Request): cond_set_value(prod_item, 'url', prod_url.url) # Tentative. yield prod_url else: # Another request is necessary to complete the product. url = urlparse.urljoin(response.url, prod_url) cond_set_value(prod_item, 'url', url) # Tentative. yield Request(url, callback=self.parse_product, meta={'product': prod_item})