class OrientaltradingProductsSpider(BaseProductsSpider): name = 'orientaltrading_products' allowed_domains = ['orientaltrading.com', "www.orientaltrading.com"] start_urls = [] SEARCH_URL = "http://www.orientaltrading.com/web/search/searchMain?Ntt={search_term}" PAGINATE_URL = "http://www.orientaltrading.com/web/search/searchMain?Nrpp=64&No={nao}&Ntt={search_term}" CURRENT_NAO = 0 PAGINATE_BY = 64 # 64 products TOTAL_MATCHES = None # for pagination REVIEW_URL = "http://orientaltrading.ugc.bazaarvoice.com/0713-en_us/{product_id}" \ "/reviews.djs?format=embeddedhtml&page={index}&" VARIANT_PRODUCT = 'http://www.orientaltrading.com/web/browse/processProductsCatalog' #use_proxies = True def __init__(self, *args, **kwargs): self.br = BuyerReviewsBazaarApi(called_class=self) super(OrientaltradingProductsSpider, self).__init__(site_name=self.allowed_domains[0], *args, **kwargs) def _parse_single_product(self, response): return self.parse_product(response) def parse_product(self, response): meta = response.meta.copy() product = meta.get('product', SiteProductItem()) reqs = [] meta['reqs'] = reqs # Parse locate locale = 'en_US' cond_set_value(product, 'locale', locale) # Parse title title = self.parse_title(response) cond_set(product, 'title', title) # Parse image image = self.parse_image(response) cond_set(product, 'image_url', image) # Parse sku sku = self.parse_sku(response) cond_set_value(product, 'sku', sku) # Parse reseller_id cond_set_value(product, "reseller_id", sku) # Parse price price = self.parse_price(response) cond_set_value(product, 'price', price) # Parse description description = self.parse_description(response) cond_set(product, 'description', description) product['related_products'] = self.parse_related_product(response) otv = OrientaltradingVariants() otv.setupSC(response) _variants = otv._variants() if _variants: product['variants'] = _variants # reqs = self.parse_variants(response, reqs) # Parse reviews reqs.append( Request(url=self.REVIEW_URL.format( product_id=product['sku'].replace('/', '_'), index=0), dont_filter=True, callback=self.parse_buyer_reviews, meta=meta)) if reqs: return self.send_next_request(reqs, response) else: return product def clear_text(self, str_result): return str_result.replace("\t", "").replace("\n", "").replace( "\r", "").replace(u'\xa0', ' ').strip() def send_next_request(self, reqs, response): """ Helps to handle several requests """ req = reqs.pop(0) new_meta = response.meta.copy() if reqs: new_meta["reqs"] = reqs return req.replace(meta=new_meta) def parse_related_product(self, response): related_prods = [] urls = response.xpath( '//div[contains(@class, "ymal-content-wrapper")]/p/a/@href' ).extract() titles = response.xpath( '//div[contains(@class, "ymal-content-wrapper")]/p/a/text()' ).extract() # Title for title, url in zip(titles, urls): if url and title: related_prods.append(RelatedProduct(title=title, url=url)) related_products = {} if related_prods: related_products['you may also like'] = related_prods return related_products def parse_title(self, response): title = response.xpath( '//meta[contains(@property, "og:title")]/@content').extract() return title def parse_image(self, response): img = response.xpath( '//meta[contains(@property, "og:image")]/@content').extract() return img def parse_description(self, response): description = response.xpath( '//div[contains(@class, "pd-text-bloc")] | //p[contains(@class, "pd-text-bloc")]' ).extract() if description: return description else: return '' def parse_sku(self, response): sku = response.xpath( '//input[contains(@id, "productsku")]/@value').extract() if sku: return sku[0] def parse_productid(self, response): model = response.xpath( '//input[contains(@id, "productId")]/@value').extract() if model: return model[0] def parse_price(self, response): price = response.xpath( '//p[contains(@id, "pd-price")]/text()').extract() if price: price = self.clear_text(price[0].replace('NOW', '').replace('$', '')) return Price(price=price, priceCurrency="USD") else: return Price(price=0.00, priceCurrency="USD") """ def parse_variants(self, response, reqs): select_variants = response.xpath('//fieldset[contains(@class, "select-options")]/select') if select_variants: OTC_CSRFTOKEN = response.xpath('//input[contains(@name, "OTC_CSRFTOKEN")]/@value').extract() prefix = response.xpath('//input[contains(@id, "prefix")]/@value').extract() productId = response.xpath('//input[contains(@id, "productId")]/@value').extract() parentSku = response.xpath('//input[contains(@id, "parentSku")]/@value').extract() demandPrefix = response.xpath('//input[contains(@id, "demandPrefix")]/@value').extract() pznComponentIndex = response.xpath('//input[contains(@id, "pznComponentIndex")]/@value').extract() pznHiddenData = response.xpath('//input[contains(@id, "pznHiddenData")]/@value').extract() pznImageName = response.xpath('//input[contains(@id, "pznImageName")]/@value').extract() destinationDisplayJSP = response.xpath('//input[contains(@name, "destinationDisplayJSP")]/@value').extract() requestURI = response.xpath('//input[contains(@name, "requestURI")]/@value').extract() numberOfAttributes = response.xpath('//input[contains(@id, "numberOfAttributes")]/@value').extract() categoryId = response.xpath('//input[contains(@id, "categoryId")]/@value').extract() mode = response.xpath('//input[contains(@id, "mode")]/@value').extract() quantity = response.xpath('//input[contains(@name, "quantity")]/@value').extract() params = {'OTC_CSRFTOKEN': OTC_CSRFTOKEN[0], 'categoryId': categoryId[0], 'demandPrefix': demandPrefix[0], 'destinationDisplayJSP': destinationDisplayJSP[0], 'mode': mode[0], 'numberOfAttributes': numberOfAttributes[0], 'parentSku': parentSku[0], 'prefix': prefix[0], 'productId': productId[0], 'pznComponentIndex': pznComponentIndex[0], 'pznHiddenData': pznHiddenData[0], 'pznImageName': pznImageName[0], 'quantity': quantity[0], 'requestURI': requestURI[0], 'sku': '', } for v in select_variants: name = v.xpath('@name').extract() options = v.xpath('option/@value').extract() for opt in options: if opt: # TODO: get variant sku for params['sku'] # url = 'http://www.orientaltrading.com/rest/ajax/' # post_data = {'formData': "{\"sku\":\"%s\",\"uniqueIdentifier\":\"\",\"nameArray\":[\"%s\"]," # "\"valueArray\":[\"%s\"],\"command\":\"AttributeSkuLookup\"}" % (sku, name[0], opt[0]), # 'requestURI': "/" # } # reqs.append(FormRequest(url=url, formdata=post_data, callback=self.get_sku_attribute)) params[name[0]] = opt reqs.append(FormRequest(url=self.VARIANT_PRODUCT, formdata=params, callback=self.parse_variants_info)) return reqs """ """ def parse_variants_info(self, response): meta = response.meta.copy() product = response.meta['product'] reqs = meta.get('reqs', []) sku = self.parse_sku(response) price = self.parse_price(response) if reqs: return self.send_next_request(reqs, response) else: return product """ def get_sku_attribute(self, response): meta = response.meta.copy() product = response.meta['product'] reqs = meta.get('reqs', []) jsondata = json.loads(response.body_as_unicode()) # {"uniqueIdentifier":"","parentSku":"13578611","attributeSku":"13582742"} new_sku = jsondata['attributeSku'] if reqs: return self.send_next_request(reqs, response) else: return product def parse_buyer_reviews(self, response): meta = response.meta.copy() product = response.meta['product'] reqs = meta.get('reqs', []) product['buyer_reviews'] = self.br.parse_buyer_reviews_per_page( response) if reqs: return self.send_next_request(reqs, response) else: return product def _scrape_total_matches(self, response): data = re.findall(r'site_search_results: "(.+)"', response.body_as_unicode()) if data: totals = data[0] if totals.isdigit(): if not self.TOTAL_MATCHES: self.TOTAL_MATCHES = int(totals) return int(totals) else: return 0 def _scrape_product_links(self, response): for link in response.xpath( '//div[contains(@id, "tableSearchResultsPhoto")]/a/@href' ).extract(): yield link, SiteProductItem() # def _get_nao(self, url): # nao = re.search(r'pn=(\d+)', url) # if not nao: # return # return int(nao.group(1)) # # def _replace_nao(self, url, new_nao): # current_nao = self._get_nao(url) # if current_nao: # return re.sub(r'nao=\d+', 'pn=' + str(new_nao), url) # else: # return url + '&pn=' + str(new_nao) def _scrape_next_results_page_link(self, response): if self.TOTAL_MATCHES is None: self.log('No "next result page" link!') return if self.CURRENT_NAO > self.TOTAL_MATCHES + self.PAGINATE_BY: return # it's over self.CURRENT_NAO += self.PAGINATE_BY return Request(self.PAGINATE_URL.format( search_term=response.meta['search_term'], nao=str(self.CURRENT_NAO)), callback=self.parse, meta=response.meta)
class DellProductSpider(BaseProductsSpider): name = 'dell_products' allowed_domains = ["dell.com", "recs.richrelevance.com"] handle_httpstatus_list = [404, 403, 502, 520] SEARCH_URL = "http://pilot.search.dell.com/{search_term}" REVIEW_URL = "http://reviews.dell.com/2341_mg/{product_id}/reviews.htm?format=embedded" VARIANTS_URL = "http://www.dell.com/api/configService.svc/postmoduleoverrides/json" VARIANTS_DATA = { 'c': 'us', 'l': 'en', 's': 'dhs', 'cs': '19', 'moduleTemplate': 'products/ProductDetails/mag/config_popup_mag', 'modErrorTemplate': 'products/module_option_validation', 'resultType': 'SingleModule', 'productCode': 'undefined' } # there are two types of product pages, each of them requires different related products processing RELATED_PROD_URL_V1 = ( "http://recs.richrelevance.com/rrserver/p13n_generated.js?" "pt=|item_page.mag_syspdpoc1|item_page.mag_syspdpoc2|item_page.mag_syspdpoc3|item_page.mag_syspdpoc4|item_page.mag_syspdpoc5&" "a=usdhsa5d5af7012d61fd1&rid=us_19_en_dhs&sgs=|us_19_en_dhs:us_19_en_dhs&flv=15.0.0&" "s=undefined{date}&n={n}&chi={chi}&ts={ts}&p={p}") RELATED_PROD_URL_V2 = ( "http://recs.richrelevance.com/rrserver/p13n_generated.js?" "pt=|item_page.storm_snp_pdp1|item_page.storm_snp_pdp2|item_page.storm_snp_pdp3|item_page.storm_snp_pdp4|item_page.storm_snp_pdp5&" "sgs=|us_04_en_bsd:us_04_en_bsd&rid=us_04_en_bsd&flv=11.2.999&l=1&" "u=ykOA15fokzi417dpJeveUF65A0NwWJeGhQ6pvWEfbCuYOurQKpNgzVVXCdsYKqf4&" "s=ykOA15fokzi417dpJeveUF65A0NwWJeGhQ6pvWEfbCuYOurQKpNgzVVXCdsYKqf4{date}&" "a=usbsda5d5af7012d61fd1&ts={ts}&p={p}") def __init__(self, sort_mode=None, *args, **kwargs): from scrapy.conf import settings settings.overrides['DEPTH_PRIORITY'] = 1 settings.overrides[ 'SCHEDULER_DISK_QUEUE'] = 'scrapy.squeue.PickleFifoDiskQueue' settings.overrides[ 'SCHEDULER_MEMORY_QUEUE'] = 'scrapy.squeue.FifoMemoryQueue' self.quantity = kwargs.get('quantity', 1000) # default is 1000 self.br = BuyerReviewsBazaarApi(called_class=self) super(DellProductSpider, self).__init__(site_name=self.allowed_domains[0], *args, **kwargs) def start_requests(self): for st in self.searchterms: yield Request( self.url_formatter.format( self.SEARCH_URL, search_term=urllib.quote(st.encode('utf-8')), ), meta={ 'search_term': st, 'remaining': self.quantity }, ) if self.product_url: prod = SiteProductItem() prod['is_single_result'] = True yield Request(self.product_url, self._parse_single_product, meta={'product': prod}) def _parse_single_product(self, response): return self.parse_product(response) def _get_product_links_from_serp(self, driver): results = [] links = driver.find_elements_by_xpath( '//h4/../../a[contains(@href, "/")]') for l in links: href = l.get_attribute('href') if href: if not href.startswith('http'): href = urlparse.urljoin( 'http://' + self.allowed_domains[0], href) results.append(href) return results def _is_product_page(self, response): return 'is_product_page' in response.meta def _init_webdriver(self): from selenium import webdriver from selenium.webdriver.remote.remote_connection import RemoteConnection RemoteConnection.set_timeout(30) driver = webdriver.Firefox() driver.set_window_size(1280, 1024) driver.set_page_load_timeout(60) driver.set_script_timeout(60) return driver def parse(self, response): if not self._is_product_page(response): product_links = [] # scrape "quantity" products display = Display(visible=0, size=(1280, 1024)) display.start() driver = self._init_webdriver() driver.get(response.url) time.sleep(6) # let AJAX finish new_meta = response.meta.copy() # get all products we need (or till the "show more products" button exists) paging_button = '//button[contains(@id, "paging-button")]' num_of_errors = 0 while driver.find_elements_by_xpath(paging_button): try: button = driver.find_elements_by_xpath(paging_button) button[0].click() time.sleep(4) product_links = self._get_product_links_from_serp(driver) if len(product_links) > self.quantity: break print 'Collected %i product links' % len(product_links) self.log('Collected %i product links' % len(product_links)) except Exception as e: print str(e) self.log('Error while doing pagination: %s' % str(e), WARNING) num_of_errors += 1 if num_of_errors > 10: self.log('Too many webdriver errors', ERROR) driver.quit() display.stop() return #driver.save_screenshot('/tmp/1.png') new_meta['is_product_page'] = True for i, product_link in enumerate(product_links): new_meta['_ranking'] = i + 1 yield Request(product_link, meta=new_meta, callback=self.parse_product) driver.quit() try: display.stop() except Exception as e: self.log('Exception on display.stop(): [%s]' % str(e)) @staticmethod def _parse_price(response): dell_price = response.xpath('//*[contains(text(), "Dell Price")]') dell_price = re.search( '\$([\d,]+\.\d+)', ''.join(dell_price.xpath('./..//text()').extract())) if dell_price: dell_price = dell_price.group(1) price = Price(price=dell_price, priceCurrency='USD') return price price = response.xpath('//*[contains(@name, "pricing_sale_price")]' '[contains(text(), "$")]//text()').extract() if not price: price = response.xpath( '//*[contains(@name, "pricing_retail_price")]' '[contains(text(), "$")]//text()').extract() if price: price = Price(price=price[0].strip().replace('$', ''), priceCurrency='USD') return price @staticmethod def _parse_image(response): img_src = response.xpath( '//*[contains(@id, "product_main_image")]' '//img[contains(@src, ".jp")]/@src').extract() if not img_src: img_src = response.xpath( '//*[contains(@class, "oneImageUp")]' '//img[contains(@src, ".jp")]/@src').extract() if not img_src: img_src = response.xpath( '//*[contains(@class, "leftRightMainImg")]' '//img[contains(@src, ".jp")]/@src').extract() if not img_src: img_src = response.xpath( '//*[contains(@class, "oneImageUp")]' '//img[contains(@data-original, ".jp")]/@data-original' ).extract() if img_src: return img_src[0] @staticmethod def _parse_brand(response, prod_title): # <meta itemprop="brand" content = "DELL"/> brand = response.xpath( '//meta[contains(@itermprop, "brand")]/@content').extract() if not brand: brand = response.xpath( '//a[contains(@href, "/brand.aspx")]/img/@alt').extract() if brand: return brand[0].title() if prod_title: brand = guess_brand_from_first_words(prod_title) if not brand: prod_title = prod_title.replace('New ', '').strip() brand = guess_brand_from_first_words(prod_title) if brand: return brand @staticmethod def _parse_description(response): desc = response.xpath('//*[@id="cntTabsCnt"]').extract() if not desc: desc = response.xpath( './/*[@id="AnchorZone3"]' '//div[not(contains(@class, "anchored_returntotop"))]' ).extract() if desc: return desc[0] def _related_products(self, response): results = [] rps = response.xpath( '//*[contains(@class, "psItemDescription")]//' 'div[contains(@class, "psTeaser")]//a[contains(@href, "productdetail.aspx")]' ) for rp in rps: results.append( RelatedProduct( rp.xpath('text()').extract()[0].strip(), rp.xpath('@href').extract() [0].strip())) # TODO: check if it's a valid format # TODO: scrape dynamic related products return results def parse_buyer_reviews(self, response): product = response.meta['product'] buyer_reviews = self.br.parse_buyer_reviews_per_page(response) product['buyer_reviews'] = buyer_reviews yield product def _get_stock_status(self, response, product): oos_element = response.xpath( '//a[contains(@class, "smallBlueBodyText")]' '[contains(@href, "makeWin")]//text()').extract() if oos_element: oos_element = oos_element[0].lower() if ('temporarily out of stock' in oos_element or 'pre-order' in oos_element): product['is_out_of_stock'] = True return product if 'limited supply available' in oos_element: product['is_out_of_stock'] = False product['limited_stock'] = LimitedStock(is_limited=True, items_left=-1) return product @staticmethod def _get_product_id(response): prod_id = re.findall(':productdetails:([\da-zA-Z\-\.]{1,50})\",', response.body_as_unicode()) if prod_id: return prod_id[0] def parse_product(self, response): prod = response.meta.get('product', SiteProductItem()) prod['_subitem'] = True _ranking = response.meta.get('_ranking', None) prod['ranking'] = _ranking prod['url'] = response.url cond_set(prod, 'title', response.css('h1 ::text').extract()) prod['price'] = DellProductSpider._parse_price(response) prod['image_url'] = DellProductSpider._parse_image(response) prod['description'] = DellProductSpider._parse_description(response) prod['brand'] = DellProductSpider._parse_brand(response, prod.get('title', '')) prod['related_products'] = self._related_products(response) response.meta['product'] = prod is_links, variants = self._parse_variants(response) if is_links: yield variants.pop(0) else: cond_set_value(prod, 'variants', self._collect_variants_from_dict(variants)) if 'This product is currently unavailable.' in response.body_as_unicode( ): prod['is_out_of_stock'] = True else: yield self._get_stock_status(response, prod) # this should be OOS field meta = {'product': prod} prod_id = self._get_product_id(response) if prod_id: # first page type if response.css('#bazaarVoice').extract(): meta.update({'br_page_type': 1}) yield Request( # reviews request url=self.REVIEW_URL.format(product_id=prod_id), dont_filter=True, callback=self.parse_buyer_reviews, meta=meta) buyer_reviews_iframe_src = response.xpath( '//iframe[contains(@src,"reviews.htm")]/@src').extract() if buyer_reviews_iframe_src: # second page type meta.update({'br_page_type': 2}) yield Request( # reviews request url=buyer_reviews_iframe_src[0].replace('format=noscript', ''), dont_filter=True, callback=self.parse_buyer_reviews, meta=meta) try: r_url, related_data = self.RELATED_PROD_URL_V1, self._collect_related_products_data_v1( response) except Exception: r_url, related_data = self.RELATED_PROD_URL_V2, self._collect_related_products_data_v2( response) yield Request( # related products request r_url.format(**related_data), callback=self._parse_related_products, meta=meta) yield prod def _collect_common_variants_data(self, response): data = self.VARIANTS_DATA.copy() _ = is_empty( response.xpath('//meta[@name="Country"]/@content').extract()) if _: data['c'] = _ _ = is_empty( response.xpath('//meta[@name="Language"]/@content').extract()) if _: data['l'] = _ _ = is_empty( response.xpath('//meta[@name="Segment"]/@content').extract()) if _: data['s'] = _ _ = is_empty( response.xpath('//meta[@name="CustomerSet"]/@content').extract()) if _: data['cs'] = _ _ = is_empty( response.xpath('//meta[@name="currentOcId"]/@content').extract()) if _: data['oc'] = _ else: self.log('No "OC" and/or "modId data found" <%s>' % response.url, WARNING) return None return data def _collect_specific_variants_data(self, variant, common_data): data = common_data.copy() oc = data.get('oc') if not oc: self.log('No OC data', ERROR) uniq_id = is_empty( variant.xpath( '//input[@value="%s"][contains(@id, "OrderCode")]/@id' % oc).extract()) uniq_id = uniq_id.replace('OrderCode', '') mod_id = is_empty( variant.xpath('.//span[contains(@class,"spec~%s~")]/@class' % uniq_id).extract()) mod_id = mod_id.split('~')[-1] data['modId'] = mod_id data['uiParameters'] = 'mainModuleId=%s&uniqueId=%s' % (mod_id, uniq_id) return data def _collect_variants_from_dict(self, variants): if not variants: return None max_options = 4 _variants = OrderedDict() keys = sorted(variants.keys()[:max_options]) for tmp in keys: _variants[tmp] = variants[tmp] options = product(*_variants.values()[:max_options]) data = [] for option in options: tmp = {} for i, key in enumerate(keys): tmp[key] = option[i] data.append( dict(in_stock=None, price=None, selected=None, properties=tmp)) return data def _parse_variant_data(self, response): json_resp = hjson.loads(response.body_as_unicode()) html = json_resp['ModulesHtml'] html = Selector(text=html) add_requests = response.meta.get('additional_requests') variants = response.meta['variants'] cur_var = response.meta['cur_variant'] choices = html.css('.catContent .optDescription::text').extract() variants[cur_var] = choices if add_requests: next_request = add_requests.pop(0) return next_request vs = self._collect_variants_from_dict(variants) prod = response.meta['product'] prod['variants'] = vs return prod def _parse_variants(self, response): variants_exist = bool(response.css('#Configurations').extract()) if variants_exist: common_req_params = self._collect_common_variants_data(response) variants_names = response.xpath( '//div[contains(@class, "specContent")]') data = {} additional_requests = [] for v_n in variants_names: k = is_empty( v_n.xpath( 'normalize-space(preceding-sibling::div[@class="specTitle"][1]/h5/text())' ).extract()) v = ' '.join(v_n.xpath('span/text()').extract()) is_ajax = bool(v_n.xpath('div[@class="dropdown"]').extract()) if is_ajax: form_data = self._collect_specific_variants_data( v_n, common_req_params) meta = response.meta.copy() meta['variants'] = data meta['cur_variant'] = k meta['additional_requests'] = additional_requests meta['product'] = response.meta['product'] additional_requests.append( FormRequest(self.VARIANTS_URL, callback=self._parse_variant_data, formdata=form_data, meta=meta)) else: data[k] = [v] if additional_requests: return True, additional_requests else: return False, data return None, None def _collect_related_products_data_v1(self, response): data = dict() cur_date = datetime.now() js_node = response.xpath( '//div[@id="mbox_default"]/following-sibling::script[1]') js_data = js_node.xpath('following-sibling::script[1]/text()').re( 'profile = (\{.*\})') js_data = hjson.loads(js_data[0]) data['p'] = is_empty( response.css('meta[name=currentOcId]::attr(content)').extract()) data['date'] = cur_date.today().strftime('%Y%m%d') data['ts'] = '%s000' % int(time.mktime(cur_date.timetuple())) data['n'] = js_data['catid'] data['chi'] = is_empty( js_node.xpath('text()').re("'profile.catid=(.*?)'")) return data def _collect_related_products_data_v2(self, response): data = dict() js_data = response.xpath( 'normalize-space(/html/head/script[@type="text/javascript"][1]/text())' ).re('\{.*\}') js_data = hjson.loads(js_data[0]) cur_date = datetime.now() data['date'] = cur_date.today().strftime('%Y%m%d') data['ts'] = '%s000' % int(time.mktime(cur_date.timetuple())) data['p'] = js_data['CJ']['ORDERCODE'].lower() return data def _parse_related_products(self, response): prod = response.meta['product'] html = re.search(r"html:'(.+?)'\}\]\},", response.body_as_unicode()) if not html: return prod html = Selector(text=html.group(1)) key_name = is_empty(html.css('.rrStrat::text').extract()) items = html.css('.rrRecs > ul > li') rel_prods = [] for item in items: title = is_empty(item.css('.rrItemName > a ::text').extract()) url = is_empty(item.css('a.rrLinkUrl::attr(href)').extract()) url = urlparse.urlparse(url) qs = urlparse.parse_qs(url.query) url = is_empty(qs['ct']) rel_prods.append(RelatedProduct(title=title, url=url)) prod['related_products'] = {key_name: rel_prods} return prod
class DockersProductsSpider(BaseValidator, BaseProductsSpider): name = 'dockers_products' allowed_domains = ["dockers.com", "www.dockers.com"] start_urls = [] settings = DockersValidatorSettings SEARCH_URL = "http://www.dockers.com/US/en_US/search?Ntt={search_term}" # TODO: ordering PAGINATE_URL = ('http://www.dockers.com/US/en_US/includes/searchResultsScroll/?nao={nao}' '&url=%2FUS%2Fen_US%2Fsearch%3FD%3D{search_term}%26Dx' '%3Dmode%2Bmatchall%26N%3D4294961104%2B4294961101%2B4294965619%26Ntk' '%3DAll%26Ntt%3D{search_term}%26Ntx%3Dmode%2Bmatchall') CURRENT_NAO = 0 PAGINATE_BY = 12 # 12 products TOTAL_MATCHES = None # for pagination total_matches = None REVIEW_URL = "http://dockers.ugc.bazaarvoice.com/2080-en_us/{product_id}" \ "/reviews.djs?format=embeddedhtml&page={index}&" RELATED_PRODUCT = "https://levis.tt.omtrdc.net/m2/levis/mbox/ajax?" \ "mboxHost=www.dockers.com" \ "&mboxSession=1481449902450-970396" \ "&mboxCount=1" \ "&entity.id={product_id}" \ "&entity.categoryId={product_categories}" \ "&mbox=target-global-mbox" \ "&mboxId=0" \ "&mboxURL={product_url}" \ "&mboxReferrer=http://www.dockers.com/" \ "&mboxVersion=60" use_proxies = True handle_httpstatus_list = [404] def __init__(self, *args, **kwargs): self.br = BuyerReviewsBazaarApi(called_class=self) super(DockersProductsSpider, self).__init__( site_name=self.allowed_domains[0], *args, **kwargs) def _parse_single_product(self, response): return self.parse_product(response) def _init_firefox(self, proxy): from selenium import webdriver from selenium.webdriver.remote.remote_connection import RemoteConnection RemoteConnection.set_timeout(30) profile = webdriver.FirefoxProfile() #profile.set_preference("general.useragent.override", self.user_agent) profile.set_preference('intl.accept_languages', 'en-US') profile.set_preference("network.proxy.type", 1) # manual proxy configuration if proxy: # we assume only http proxies are accepted, format: http://host:port proxy, port = proxy.replace('http://', '').split(':') profile.set_preference("network.proxy.http", proxy) profile.set_preference("network.proxy.http_port", int(port)) profile.update_preferences() driver = webdriver.Firefox(profile) driver.set_window_size(1280, 1024) driver.set_page_load_timeout(60) driver.set_script_timeout(60) return driver def _is_product_page(self, response): return 'is_product_page' in response.meta def _get_product_links_from_serp(self, driver): result = [] for l in driver.find_elements_by_xpath( '//li[contains(@class, "product-tile")]' '//a[contains(@rel, "product")]' ): href = l.get_attribute('href') if href: if not href.startswith('http'): href = urlparse.urljoin('http://' + self.allowed_domains[0], href) result.append(href) return result @staticmethod def last_six_digits_the_same(lst): print lst if len(lst) < 7: return return len(set(lst[-6:-1])) == 1 # if all elements are the same, set's length will be 1 def parse(self, response): proxy = response.request.meta.get('proxy', None) if not self._is_product_page(response): self.total_matches = self._scrape_total_matches(response) display = Display(visible=0, size=(1280, 1024)) display.start() product_links = [] # scrape "quantity" products driver = self._init_firefox(proxy=proxy) try: driver.get('http://www.dockers.com/US/en_US/') except Exception as e: print(str(e)) self.log(str(e)) driver.find_element_by_name('Ntt').send_keys(self.searchterms[0] + '\n') time.sleep(10) # let AJAX finish new_meta = response.meta.copy() # get all products we need (scroll down) collected_products_len = [] num_of_errors = 0 while True: try: driver.execute_script("scrollTo(0,50000)") time.sleep(10) product_links = self._get_product_links_from_serp(driver) collected_products_len.append(len(product_links)) if self.last_six_digits_the_same(collected_products_len): break # last six iterations collected equal num of products if len(product_links) > self.quantity: break print 'Collected %i product links' % len(product_links) self.log('Collected %i product links' % len(product_links)) self.log('Statistics: %s' % report_statistics()) except Exception as e: print str(e) self.log('Error while doing pagination %s' % str(e), WARNING) num_of_errors += 1 if num_of_errors > 10: self.log('Too many webdriver errors', ERROR) driver.quit() display.stop() return #driver.save_screenshot('/tmp/1.png') new_meta['is_product_page'] = True for i, product_link in enumerate(product_links): new_meta['_ranking'] = i+1 yield Request(product_link, meta=new_meta, callback=self.parse_product) driver.quit() display.stop() def parse_product(self, response): meta = response.meta.copy() product = meta.get('product', SiteProductItem()) if response.status == 404 or "www.dockers.com/US/en_US/error" in response.url: product.update({"not_found": True}) product.update({"no_longer_available": True}) product.update({"locale": 'en-US'}) return product else: product.update({"no_longer_available": False}) reqs = [] meta['reqs'] = reqs product['ranking'] = response.meta.get('_ranking', None) product['total_matches'] = self.total_matches product['url'] = response.url product['site'] = self.allowed_domains[0] product['search_term'] = self.searchterms[0] if self.searchterms else None product['scraped_results_per_page'] = product['results_per_page'] = self.PAGINATE_BY # product id self.product_id = is_empty(response.xpath('//meta[@itemprop="model"]/@content').extract()) # product data in json self.js_data = self.parse_data(response) # Parse locate locale = 'en_US' cond_set_value(product, 'locale', locale) # Parse model cond_set_value(product, 'model', self.product_id) reseller_id_regex = "p\/([^\/&?\.\s]+)" reseller_id = re.findall(reseller_id_regex, response.url) reseller_id = reseller_id[0] if reseller_id else None cond_set_value(product, 'reseller_id', reseller_id) # Parse title title = self.parse_title(response) cond_set(product, 'title', title) # Parse image image = self.parse_image(response) cond_set_value(product, 'image_url', image) # Parse brand brand = self.parse_brand(response) cond_set_value(product, 'brand', brand) # Parse upc upc = self.parse_upc(response) cond_set_value(product, 'upc', upc) # Parse sku sku = self.parse_sku(response) cond_set_value(product, 'sku', sku) # Parse description description = self.parse_description(response) cond_set_value(product, 'description', description) # Parse price price = self.parse_price(response) cond_set_value(product, 'price', price) # Parse variants variants = self._parse_variants(response) product['variants'] = variants # Parse product_categories self.product_categories = self._extract_categories(response.body_as_unicode()) response.meta['marks'] = {'1': 0, '2': 0, '3': 0, '4': 0, '5': 0} real_count = is_empty(re.findall(r'<span itemprop="reviewCount">(\d+)<\/span>', response.body_as_unicode())) response.meta['product'] = product meta = response.meta if real_count: # Parse buyer reviews if int(real_count) > 8: for index, i in enumerate(xrange(9, int(real_count) + 1, 30)): reqs.append( Request( url=self.REVIEW_URL.format(product_id=self.product_id, index=index+2), dont_filter=True, callback=self.parse_buyer_reviews, meta=meta ) ) reqs.append( Request( url=self.REVIEW_URL.format(product_id=self.product_id, index=0), dont_filter=True, callback=self.parse_buyer_reviews, meta=meta )) if reqs: return self.send_next_request(reqs, response) return product def _parse_variants(self, response): """ Parses product variants. """ dk = DockersVariants() dk.setupSC(response) variants = dk._variants() return variants def parse_buyer_reviews(self, response): meta = response.meta.copy() buyer_reviews_per_page = self.br.parse_buyer_reviews_per_page(response) for k, v in buyer_reviews_per_page['rating_by_star'].iteritems(): response.meta['marks'][k] += v product = response.meta['product'] reqs = meta.get('reqs', []) product['buyer_reviews'] = BuyerReviews( num_of_reviews=buyer_reviews_per_page['num_of_reviews'], average_rating=buyer_reviews_per_page['average_rating'], rating_by_star=response.meta['marks'] ) # Updated related product url, previous res-x doesn't work product_id = self.product_id + 'US' url = self.RELATED_PRODUCT.format(product_id=product_id, product_categories=self.product_categories, product_url=product.get('url')) reqs.append( Request( url=url, dont_filter=True, callback=self.parse_related_product, meta=meta )) return self.send_next_request(reqs, response) def send_next_request(self, reqs, response): """ Helps to handle several requests """ req = reqs.pop(0) new_meta = response.meta.copy() if reqs: new_meta["reqs"] = reqs return req.replace(meta=new_meta) def parse_brand(self, response): brand = is_empty(response.xpath( '//meta[@itemprop="brand"]/@content').extract()) return brand def parse_title(self, response): title = response.xpath( '//meta[contains(@property,"og:title")]/@content').extract() if not title: title = response.xpath( '//meta[contains(@name,"og:title")]/@content').extract() return title def parse_data(self, response): data = re.findall(r'var buyStackJSON = \'(.+)\'; ', response.body_as_unicode()) if data: data = re.sub(r'\\(.)', r'\g<1>', data[0]) try: js_data = json.loads(data) except: return return js_data def parse_image(self, response): if self.js_data: try: image = self.js_data['colorid'][self.product_id]['gridUrl'] except: return return image def parse_related_product(self, response): product = response.meta['product'] text = self._extract_related_products_json(response.body_as_unicode()) related_products = self._build_related_products_array(text, product) if related_products: product['related_products'] = {} product['related_products']['buyers_also_bought'] = related_products return product def parse_description(self, response): if self.js_data: try: description = self.js_data['colorid'][self.product_id]['name'] except: return return description def parse_upc(self, response): if self.js_data: for v in self.js_data['sku'].values(): upc = v['upc'] upc = upc[-12:] if len(upc) < 12: count = 12-len(upc) upc = '0'*count+upc return upc def parse_sku(self, response): if self.js_data: for v in self.js_data['sku'].values(): skuid = v['skuid'] return skuid def parse_price(self, response): if self.js_data: price = self.js_data['colorid'][self.product_id]['price'] for price_data in price: if price_data['il8n'] == 'now': price = price_data['amount'] currency = is_empty(re.findall(r'currency":"(\w+)"', response.body_as_unicode())) if price and currency: price = Price(price=price, priceCurrency=currency) else: price = Price(price=0.00, priceCurrency="USD") return price def _scrape_total_matches(self, response): totals = response.css('.productCount ::text').extract() if totals: totals = totals[0].replace(',', '').replace('.', '').strip() if totals.isdigit(): if not self.TOTAL_MATCHES: self.TOTAL_MATCHES = int(totals) return int(totals) def _scrape_product_links(self, response): for link in response.xpath( '//li[contains(@class, "product-tile")]' '//a[contains(@rel, "product")]/@href' ).extract(): yield link, SiteProductItem() def _get_nao(self, url): nao = re.search(r'nao=(\d+)', url) if not nao: return return int(nao.group(1)) def _replace_nao(self, url, new_nao): current_nao = self._get_nao(url) if current_nao: return re.sub(r'nao=\d+', 'nao='+str(new_nao), url) else: return url+'&nao='+str(new_nao) def _scrape_next_results_page_link(self, response): if self.TOTAL_MATCHES is None: self.log('No "next result page" link!') return if self.CURRENT_NAO > self.TOTAL_MATCHES+self.PAGINATE_BY: return # it's over self.CURRENT_NAO += self.PAGINATE_BY return Request( self.PAGINATE_URL.format( search_term=response.meta['search_term'], nao=str(self.CURRENT_NAO)), callback=self.parse, meta=response.meta ) @staticmethod def _extract_categories(body): pattern = re.compile('var\s+categoryIds\s*=\s*\'(.+?)\;') categories = pattern.search(body) return categories.group(1) if categories else None def _extract_related_products_json(self, body): pattern = re.compile('\_AT\.applyWhenReady\(\s*\[\s*({.+?})\s*\]\s*\)\s*;', re.DOTALL) related_products_json = pattern.search(body) data = related_products_json.group(1) if related_products_json else None try: data = json.loads(data).get('content') return data except Exception as e: self.log('{}'.format(e.message)) return None @staticmethod def _build_related_products_array(text, product): s = Selector(text=text) related_products = [] product_url = product.get('url') for element in s.xpath('//li[contains(@class, "imagegrid")]'): url = element.xpath('.//a/@href').extract() title = element.xpath('.//p[@class="name"]/text()').extract() if url and title: url = urlparse.urljoin(product_url, url[0]) title = title[0] related_products.append(RelatedProduct(url=url, title=title)) return related_products
class OfficedepotProductsSpider(BaseProductsSpider): name = 'officedepot_products' allowed_domains = [ "officedepot.com", "www.officedepot.com", 'bazaarvoice.com' ] start_urls = [] _extra_requests = False # settings = DockersValidatorSettings SEARCH_URL = "http://www.officedepot.com/catalog/search.do?Ntt={search_term}&searchSuggestion=true&akamai-feo=off" PAGINATE_URL = ( 'http://www.officedepot.com/catalog/search.do?Ntx=mode+matchpartialmax&Nty=1&Ntk=all' '&Ntt={search_term}&N=5&recordsPerPageNumber=24&No={nao}') CURRENT_NAO = 0 PAGINATE_BY = 24 # 24 products TOTAL_MATCHES = None # for pagination REVIEW_URL = "http://officedepot.ugc.bazaarvoice.com/2563" \ "/{product_id}/reviews.djs?format=embeddedhtml" VARIANTS_URL = 'http://www.officedepot.com/mobile/getSkuAvailable' \ 'Options.do?familyDescription={name}&sku={sku}&noLogin=true' QA_URL = "http://officedepot.ugc.bazaarvoice.com/answers/2563/product/{product_id}/questions.djs?format=embeddedhtml" # # RELATED_PRODUCT = "http://www.res-x.com/ws/r2/Resonance.aspx?" \ # "appid=dockers01&tk=187015646137297" \ # "&ss=182724939426407" \ # "&sg=1&" \ # "&vr=5.3x&bx=true" \ # "&sc=product4_rr" \ # "&sc=product3_rr" \ # "&sc=product1_r" \ # "r&sc=product2_rr" \ # "&ev=product&ei={product_id}" \ # "&no=20" \ # "&language=en_US" \ # "&cb=certonaResx.showResponse" \ # "&ur=http%3A%2F%2Fwww.levi.com%2FUS%2Fen_US%" \ # "2Fwomens-jeans%2Fp%2F095450043&plk=&" def __init__(self, *args, **kwargs): self.br = BuyerReviewsBazaarApi(called_class=self) # officedepot seems to have a bot protection, so we first get the cookies # and parse the site with them after that self.proxy = None self.timeout = 60 self.width = 1024 self.height = 768 self.selenium_cookies = {} self.user_agent = ( 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36' ' (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36') socket.setdefaulttimeout(60) self._get_selenium_cookies_for_main_page() if kwargs.get('scrape_variants_with_extra_requests'): self._extra_requests = True super(OfficedepotProductsSpider, self).__init__(site_name=self.allowed_domains[0], *args, **kwargs) def _prepare_driver(self, driver): driver.set_page_load_timeout(int(self.timeout)) driver.set_script_timeout(int(self.timeout)) driver.set_window_size(int(self.width), int(self.height)) def _get_selenium_cookies_for_main_page(self): from pyvirtualdisplay import Display display = Display(visible=False) display.start() driver = self._init_chromium() self._prepare_driver(driver) try: driver.get('http://' + self.allowed_domains[0]) time.sleep(10) for cookie in driver.get_cookies(): self.selenium_cookies[cookie['name']] = cookie['value'] driver.quit() except Exception as e: driver.quit() time.sleep(10) self.log( 'Error getting cookies from homepage, trying one more time: %s' % str(e)) driver.get('http://' + self.allowed_domains[0]) time.sleep(10) for cookie in driver.get_cookies(): self.selenium_cookies[cookie['name']] = cookie['value'] try: driver.quit() display.stop() except Exception as e: self.log('Error on driver & display destruction: %s' % str(e)) def _init_chromium(self): from selenium import webdriver from selenium.webdriver.remote.remote_connection import RemoteConnection RemoteConnection.set_timeout(30) chrome_flags = webdriver.DesiredCapabilities.CHROME # this is for Chrome? chrome_options = webdriver.ChromeOptions() # this is for Chromium if self.proxy: chrome_options.add_argument('--proxy-server=%s' % self.proxy_type + '://' + self.proxy) chrome_flags["chrome.switches"] = ['--user-agent=%s' % self.user_agent] chrome_options.add_argument('--user-agent=%s' % self.user_agent) executable_path = '/usr/sbin/chromedriver' if not os.path.exists(executable_path): executable_path = '/usr/local/bin/chromedriver' # initialize webdriver, open the page and make a screenshot driver = webdriver.Chrome(desired_capabilities=chrome_flags, chrome_options=chrome_options, executable_path=executable_path) return driver def _init_firefox(self): from selenium import webdriver from selenium.webdriver.remote.remote_connection import RemoteConnection RemoteConnection.set_timeout(30) profile = webdriver.FirefoxProfile() profile.set_preference("general.useragent.override", self.user_agent) profile.set_preference("network.proxy.type", 1) # manual proxy configuration if self.proxy: if 'socks' in self.proxy_type: profile.set_preference("network.proxy.socks", self.proxy.split(':')[0]) profile.set_preference("network.proxy.socks_port", int(self.proxy.split(':')[1])) else: profile.set_preference("network.proxy.http", self.proxy.split(':')[0]) profile.set_preference("network.proxy.http_port", int(self.proxy.split(':')[1])) profile.update_preferences() driver = webdriver.Firefox(profile) return driver def _parse_single_product(self, response): return self.parse_product(response) @staticmethod def _get_product_id(url): match = re.search(r'/products/(\d{2,20})/', url) if match: return match.group(1) def parse_product(self, response): meta = response.meta product = meta.get('product', SiteProductItem()) reqs = [] meta['reqs'] = reqs product['_subitem'] = True # Parse locate locale = 'en_US' cond_set_value(product, 'locale', locale) # Parse title title = self.parse_title(response) cond_set(product, 'title', title, conv=string.strip) # Parse image image = self.parse_image(response) cond_set(product, 'image_url', image) # Parse brand brand = self.parse_brand(response) cond_set_value(product, 'brand', brand) # Parse sku sku = self.parse_sku(response) cond_set_value(product, 'sku', sku) # Parse description description = self.parse_description(response) cond_set_value(product, 'description', description) # Parse price price = self.parse_price(response) cond_set_value(product, 'price', price) # Parse model model = self._parse_model(response) cond_set_value(product, 'model', model) # Parse reseller_id reseller_id = self.parse_reseller_id(response) cond_set_value(product, "reseller_id", reseller_id) # Parse is out of stock oos = self._parse_is_out_of_stock(response) cond_set_value(product, 'is_out_of_stock', oos) # Parse categories and category categories = self._parse_categories(response) cond_set_value(product, 'categories', categories) if categories: cond_set_value(product, 'category', categories[-1]) # Parse related products related_product = self._parse_related_product(response) cond_set_value(product, 'related_products', related_product) br_count = is_empty( re.findall(r'<span itemprop="reviewCount">(\d+)<\/span>', response.body_as_unicode())) meta['_br_count'] = br_count meta['product'] = product reqs.append( Request(url=self.REVIEW_URL.format( product_id=self._get_product_id(response.url)), dont_filter=True, callback=self.parse_buyer_reviews, meta=meta)) sku = is_empty(response.xpath('//input[@name="id"]/@value').extract()) name = is_empty( response.xpath('//h1[@itemprop="name"]/text()').re('(.*?),')) if sku and name and self.scrape_variants_with_extra_requests: name = urllib.quote_plus(name.strip().encode('utf-8')) reqs.append( Request(url=self.VARIANTS_URL.format(name=name, sku=sku), callback=self._parse_variants, meta=meta)) # parse questions & answers reqs.append( Request(url=self.QA_URL.format( product_id=self._get_product_id(response.url)), callback=self._parse_questions, meta=meta, dont_filter=True)) if reqs: return self.send_next_request(reqs, response) return product def parse_reseller_id(self, response): regex = "\/(\d+)" reseller_id = re.findall(regex, response.url) reseller_id = reseller_id[0] if reseller_id else None return reseller_id def _parse_questions(self, response): meta = response.meta reqs = response.meta['reqs'] product = response.meta['product'] qa = [] questions_ids_regex = """BVQAQuestionSummary.+?javascript:void.+?>([^<]+)[^"']+["']BVQAQuestionMain(\d+)(?:.+?BVQAQuestionDetails.+?div>([^<]+)?).+?BVQAElapsedTime.+?>([^<]+)""" questions_ids = re.findall(questions_ids_regex, response.body_as_unicode()) for (question_summary, question_id, question_details, question_date) in questions_ids: # Convert date format if question_date: try: from dateutil.relativedelta import relativedelta years = re.findall("(\d+?)\s+?years", question_date) years = years[0] if years else '0' years = int(years) if years.isdigit() else '0' months = re.findall("(\d+?)\s+?months", question_date) months = months[0] if months else '0' months = int(months) if months.isdigit() else '0' if not months and not years: converted_date = None else: converted_date = datetime.now() - relativedelta( years=years, months=months) converted_date = converted_date.strftime("%Y-%m-%d") except Exception as e: converted_date = None self.log( 'Failed to parse date, setting date to None {}'.format( e)) else: converted_date = None # regex to get part of response that contain all answers to question with given id text_r = "BVQAQuestion{}Answers(.+?)BVQAQuestionDivider".format( question_id) all_a_text = re.findall(text_r, response.body_as_unicode()) all_a_text = ''.join(all_a_text[0]) if all_a_text else '' answers_regex = r"Answer:.+?>([^<]+)" answers = re.findall(answers_regex, all_a_text) answers = [{'answerText': a} for a in answers] question = { 'questionDate': converted_date, 'questionId': question_id, 'questionDetail': question_details.strip() if question_details else '', 'qestionSmmary': question_summary.strip() if question_summary else '', 'answers': answers, 'totalAnswersCount': len(answers) } qa.append(question) product['all_questions'] = qa if reqs: return self.send_next_request(reqs, response) return product def clear_text(self, str_result): return str_result.replace("\t", "").replace("\n", "").replace( "\r", "").replace(u'\xa0', ' ').strip() def _parse_is_out_of_stock(self, response): oos = response.xpath('//*[@itemprop="availability"' ' and @content="http://schema.org/OutOfStock"]') return bool(oos) def _parse_model(self, response): model = response.xpath( '//*[@id="attributemodel_namekey"]/text()').extract() if model: return model[0].strip() def _parse_categories(self, response): categories = response.xpath('//*[@id="siteBreadcrumb"]//' 'span[@itemprop="name"]/text()').extract() return categories def _parse_related_product(self, response): results = [] base_url = response.url for related_product in response.xpath( '//*[@id="relatedItems"]' '//tr[contains(@class,"hproduct")]' '/td[@class="description"]/a'): name = is_empty(related_product.xpath('text()').extract()) url = is_empty(related_product.xpath('@href').extract()) if name and url: results.append( RelatedProduct(title=name, url=urlparse.urljoin(base_url, url))) return results def _parse_variants(self, response): """ Parses product variants. """ reqs = response.meta['reqs'] product = response.meta['product'] data = json.loads(response.body) variants = [] if data.get('success'): for sku in data.get('skus', []): vr = {} vr['url'] = urlparse.urljoin(response.url, sku.get('url')) vr['skuId'] = sku.get('sku') price = is_empty( re.findall('\$([\d\.]+)', sku.get('attributesDescription', ''))) if price: vr['price'] = price name = sku.get('description', '') if name: vr['properties'] = {'title': name} vr['image_url'] = sku.get('thumbnailImageUrl').split('?')[0] variants.append(vr) product['variants'] = variants if product.get('variants') and self._extra_requests: variants_urls = [p.get('url') for p in product['variants']] for var_url in variants_urls: req = Request(url=var_url, callback=self._parse_in_stock_for_variants) req.meta['product'] = product reqs.append(req) if reqs: return self.send_next_request(reqs, response) return product # parse variants one by one and set out of stock status for each variant def _parse_in_stock_for_variants(self, response): reqs = response.meta['reqs'] product = response.meta['product'] oos = self._parse_is_out_of_stock(response) for variant in product['variants']: if variant['url'] == response.url: variant['in_stock'] = not oos break if reqs: return self.send_next_request(reqs, response) return product def parse_buyer_reviews(self, response): meta = response.meta.copy() reqs = meta['reqs'] self.br.br_count = meta['_br_count'] buyer_reviews_per_page = self.br.parse_buyer_reviews_per_page(response) product = response.meta['product'] product['buyer_reviews'] = BuyerReviews(**buyer_reviews_per_page) if reqs: return self.send_next_request(reqs, response) return product def send_next_request(self, reqs, response): """ Helps to handle several requests """ req = reqs.pop(0) new_meta = response.meta.copy() if reqs: new_meta["reqs"] = reqs return req.replace(meta=new_meta) def parse_brand(self, response): brand = is_empty( response.xpath('//td[@itemprop="brand"]/@content').extract()) if not brand: brand = is_empty( response.xpath('//td[@itemprop="brand"]/text()').extract()) if brand: brand = brand.strip() return brand def parse_title(self, response): title = response.xpath( '//h1[contains(@itemprop, "name")]/text()').extract() return title def parse_data(self, response): data = re.findall(r'var MasterTmsUdo \'(.+)\'; ', response.body_as_unicode()) if data: data = re.sub(r'\\(.)', r'\g<1>', data[0]) try: js_data = json.loads(data) except: return return js_data def parse_image(self, response): img = response.xpath( '//img[contains(@id, "mainSkuProductImage")]/@src').extract() return img def parse_description(self, response): description = response.xpath( '//div[contains(@class, "sku_desc")]').extract() if description: return self.clear_text(description[0]) else: return '' def parse_sku(self, response): sku = response.xpath( '//td[contains(@id, "basicInfoManufacturerSku")]/text()').extract( ) # sku = response.xpath('//div[contains(@id, "skuValue")]/text()').extract() if sku: return self.clear_text(sku[0]) def parse_price(self, response): price = response.xpath( '//meta[contains(@itemprop, "price")]/@content').extract() currency = response.xpath( '//meta[contains(@itemprop, "priceCurrency")]/@content').extract() if price and currency: price = Price(price=price[0], priceCurrency=currency[0]) else: price = Price(price=0.00, priceCurrency="USD") return price def parse_paginate_link(self, response, nao): check_page = '&No=%s' % nao for link in response.xpath( '//a[contains(@class, "paging")]/@href').extract(): if check_page in link: u = urlparse.urlparse(link) return urlparse.urljoin('http://www.officedepot.com', u.path) def parse_category_link(self, response): categories_links = [] for link in response.xpath( '//div[contains(@class, "category_wrapper")]/a[contains(@class, "link")]/@href' ).extract(): categories_links.append(link) def _scrape_total_matches(self, response): totals = response.xpath( '//div[contains(@id, "resultCnt")]/text()').extract() if totals: totals = totals[0].replace(',', '').replace('.', '').strip() if totals.isdigit(): if not self.TOTAL_MATCHES: self.TOTAL_MATCHES = int(totals) return int(totals) def _scrape_product_links(self, response): items = response.xpath( '//div[contains(@class, "descriptionFull")]/' 'a[contains(@class, "med_txt")]/@href').extract() or response.css( '.desc_text a::attr("href")').extract() # Scraper was redirected to product page instead of search results page if not items and "officedepot.com/a/products" in response.url: prod = SiteProductItem(search_redirected_to_product=True) # TODO we may not need any data for product aside from "search_redirected_to_product" flag. # Rework if that's the case - CON-28287 req = Request(response.url, callback=self.parse_product, dont_filter=True) req.meta["remaining"] = 0 req.meta['product'] = prod yield req, prod else: for link in items: yield link, SiteProductItem() def _get_nao(self, url): nao = re.search(r'nao=(\d+)', url) if not nao: return return int(nao.group(1)) def _replace_nao(self, url, new_nao): current_nao = self._get_nao(url) if current_nao: return re.sub(r'nao=\d+', 'nao=' + str(new_nao), url) else: return url + '&nao=' + str(new_nao) def _scrape_next_results_page_link(self, response): if self.TOTAL_MATCHES is None: self.log('No "next result page" link!') # # TODO: check result by categories # return self.parse_category_link(response) return #if self.CURRENT_NAO > self.TOTAL_MATCHES+self.PAGINATE_BY: # return # all the products have been collected if self.CURRENT_NAO > self.quantity + self.PAGINATE_BY: return # num_products > quantity self.CURRENT_NAO += self.PAGINATE_BY if '/a/browse/' in response.url: # paginate in category or subcategory new_paginate_url = self.parse_paginate_link( response, self.CURRENT_NAO) if new_paginate_url: return Request(new_paginate_url, callback=self.parse, meta=response.meta, cookies=self.selenium_cookies) return Request(self.PAGINATE_URL.format( search_term=response.meta['search_term'], nao=str(self.CURRENT_NAO)), callback=self.parse, meta=response.meta, cookies=self.selenium_cookies)
class LeviProductsSpider(BaseValidator, BaseProductsSpider): name = 'levi_products' allowed_domains = ["levi.com", "www.levi.com"] start_urls = [] settings = LeviValidatorSettings SEARCH_URL = "http://www.levi.com/US/en_US/search?Ntt={search_term}" # TODO: ordering PAGINATE_URL = ( 'http://www.levi.com/US/en_US/includes/searchResultsScroll/?nao={nao}' '&url=%2FUS%2Fen_US%2Fsearch%3FD%3D{search_term}%26Dx' '%3Dmode%2Bmatchall%26N%3D4294960840%2B4294961101%2B4294965619%26Ntk' '%3DAll%26Ntt%3Ddress%26Ntx%3Dmode%2Bmatchall') CURRENT_NAO = 0 PAGINATE_BY = 12 # 12 products TOTAL_MATCHES = None # for pagination REVIEW_URL = "http://levistrauss.ugc.bazaarvoice.com/9090-en_us/" \ "{product_id}/reviews.djs?format=embeddedhtml&page={index}&" RELATED_PRODUCT = "http://www.res-x.com/ws/r2/Resonance.aspx?" \ "appid=levi01&tk=811541814822703" \ "&ss=544367773691192" \ "&sg=1&" \ "&vr=5.3x&bx=true" \ "&sc=product4_rr" \ "&sc=product3_rr" \ "&sc=product1_r" \ "r&sc=product2_rr" \ "&ev=product&ei={product_id}" \ "&no=20" \ "&language=en_US" \ "&cb=certonaResx.showResponse" \ "&ur=http%3A%2F%2Fwww.levi.com%2FUS%2Fen_US%" \ "2Fwomens-jeans%2Fp%2F095450043&plk=&" handle_httpstatus_list = [404] use_proxies = True def __init__(self, *args, **kwargs): self.br = BuyerReviewsBazaarApi(called_class=self) super(LeviProductsSpider, self).__init__(site_name=self.allowed_domains[0], *args, **kwargs) def _parse_single_product(self, response): return self.parse_product(response) def parse_product(self, response): product = response.meta.get('product', SiteProductItem()) if response.status == 404 or 'This product is no longer available' in response.body_as_unicode() \ or "www.levi.com/US/en_US/error" in response.url: product.update({"not_found": True}) product.update({"no_longer_available": True}) return product reqs = [] # product id self.product_id = is_empty( response.xpath('//meta[@itemprop="model"]/@content').extract()) # product data in json self.js_data = self.parse_data(response) # Parse locate locale = 'en_US' cond_set_value(product, 'locale', locale) # Parse model cond_set_value(product, 'model', self.product_id) # Parse title title = self.parse_title(response) cond_set(product, 'title', title) # Parse image image = self.parse_image(response) cond_set_value(product, 'image_url', image) # Parse brand brand = self.parse_brand(response) cond_set_value(product, 'brand', brand) # Parse upc upc = self.parse_upc(response) cond_set_value(product, 'upc', upc) # Parse sku sku = self.parse_sku(response) cond_set_value(product, 'sku', sku) # Parse description description = self.parse_description(response) cond_set_value(product, 'description', description) # Parse price price = self.parse_price(response) cond_set_value(product, 'price', price) reseller_id_regex = "p\/(\d+)" reseller_id = re.findall(reseller_id_regex, response.url) reseller_id = reseller_id[0] if reseller_id else None cond_set_value(product, 'reseller_id', reseller_id) try: variants = self._parse_variants(response) except KeyError: product['not_found'] = True return product # set reseller_id for variants as well for variant in variants: v_url = variant.get('url') if v_url: reseller_id = re.findall(reseller_id_regex, v_url) reseller_id = reseller_id[0] if reseller_id else None else: reseller_id = None variant['reseller_id'] = reseller_id product['variants'] = variants response.meta['marks'] = {'1': 0, '2': 0, '3': 0, '4': 0, '5': 0} real_count = is_empty( re.findall(r'<span itemprop="reviewCount">(\d+)<\/span>', response.body_as_unicode())) if real_count: # Parse buyer reviews if int(real_count) > 8: for index, i in enumerate(xrange(9, int(real_count) + 1, 30)): reqs.append( Request(url=self.REVIEW_URL.format( product_id=self.product_id, index=index + 2), dont_filter=True, callback=self.parse_buyer_reviews)) reqs.append( Request(url=self.REVIEW_URL.format(product_id=self.product_id, index=0), dont_filter=True, callback=self.parse_buyer_reviews)) if reqs: return self.send_next_request(reqs, response) return product def _parse_variants(self, response): """ Parses product variants. """ lv = LeviVariants() lv.setupSC(response) variants = lv._variants() return variants def parse_buyer_reviews(self, response): meta = response.meta.copy() buyer_reviews_per_page = self.br.parse_buyer_reviews_per_page(response) for k, v in buyer_reviews_per_page['rating_by_star'].iteritems(): response.meta['marks'][k] += v product = response.meta['product'] reqs = meta.get('reqs') product['buyer_reviews'] = BuyerReviews( num_of_reviews=buyer_reviews_per_page['num_of_reviews'], average_rating=buyer_reviews_per_page['average_rating'], rating_by_star=response.meta['marks']) if reqs: reqs.append( Request(url=self.RELATED_PRODUCT.format( product_id=self.product_id, index=0), dont_filter=True, callback=self.parse_related_product)) if reqs: return self.send_next_request(reqs, response) return product def send_next_request(self, reqs, response): """ Helps to handle several requests """ req = reqs.pop(0) new_meta = response.meta.copy() if reqs: new_meta["reqs"] = reqs return req.replace(meta=new_meta) def parse_brand(self, response): brand = is_empty( response.xpath('//meta[@itemprop="brand"]/@content').extract()) return brand def parse_title(self, response): title = response.xpath( '//meta[contains(@property, "og:title")]/@content').extract() if title: title = [title[0].replace('™', '').replace('\u2122', '')] else: title = response.xpath( '//h1[contains(@class, "title")]/text()').extract() return title def parse_data(self, response): data = re.findall(r'var buyStackJSON = \'(.+)\'; ', response.body_as_unicode()) if data: data = re.sub(r'\\(.)', r'\g<1>', data[0]) try: js_data = json.loads(data) except: return return js_data def parse_image(self, response): if self.js_data: try: image = self.js_data['colorid'][self.product_id]['gridUrl'] except: return return image def parse_related_product(self, response): related_prods = [] product = response.meta['product'] sample = response.body_as_unicode() try: sample = sample.replace(u'certonaResx.showResponse(', '') sample = sample[:-2] data = json.loads(sample) html = data['Resonance']['Response'][2]['output'] except Exception as e: self.log( 'Error during parsing related products page: {}'.format(e)) return product else: s = Selector(text=html) titles = s.xpath('//h4/text()').extract() # Title urls = s.xpath('//img/@src').extract() # Img url for title, url in zip(titles, urls): if url and title: related_prods.append(RelatedProduct(title=title, url=url)) product['related_products'] = {} if related_prods: product['related_products'][ 'buyers_also_bought'] = related_prods return product def parse_description(self, response): if self.js_data: try: description = self.js_data['colorid'][self.product_id]['name'] except: return return description def parse_upc(self, response): if self.js_data: for v in self.js_data['sku'].values(): upc = v['upc'] upc = upc[-12:] return upc def parse_sku(self, response): if self.js_data: for v in self.js_data['sku'].values(): skuid = v['skuid'] return skuid def parse_price(self, response): if self.js_data: price = self.js_data['colorid'][self.product_id]['price'] for price_data in price: if price_data['il8n'] == 'now': price = price_data['amount'] currency = is_empty( re.findall(r'currency":"(\w+)"', response.body_as_unicode())) if price and currency: price = Price(price=price, priceCurrency=currency) else: price = Price(price=0.00, priceCurrency="USD") return price def _scrape_total_matches(self, response): totals = response.css('.productCount ::text').extract() if totals: totals = totals[0].replace(',', '').replace('.', '').strip() if totals.isdigit(): if not self.TOTAL_MATCHES: self.TOTAL_MATCHES = int(totals) return int(totals) def _scrape_product_links(self, response): for link in response.xpath( '//li[contains(@class, "product-tile")]' '//a[contains(@rel, "product")]/@href').extract(): yield link, SiteProductItem() def _get_nao(self, url): nao = re.search(r'nao=(\d+)', url) if not nao: return return int(nao.group(1)) def _replace_nao(self, url, new_nao): current_nao = self._get_nao(url) if current_nao: return re.sub(r'nao=\d+', 'nao=' + str(new_nao), url) else: return url + '&nao=' + str(new_nao) def _scrape_next_results_page_link(self, response): if self.TOTAL_MATCHES is None: self.log('No "next result page" link!') return if self.CURRENT_NAO > self.TOTAL_MATCHES + self.PAGINATE_BY: return # it's over self.CURRENT_NAO += self.PAGINATE_BY return Request(self.PAGINATE_URL.format( search_term=response.meta['search_term'], nao=str(self.CURRENT_NAO)), callback=self.parse, meta=response.meta)
class HomedepotProductsSpider(BaseValidator, BaseProductsSpider): name = 'homedepot_products' allowed_domains = ["homedepot.com", "origin.api-beta.homedepot.com"] start_urls = [] settings = HomedepotValidatorSettings SEARCH_URL = "http://www.homedepot.com/s/{search_term}?NCNI-5" DETAILS_URL = "http://www.homedepot.com/p/getSkuDetails?itemId=%s" REVIEWS_URL = "http://homedepot.ugc.bazaarvoice.com/1999m/%s/" \ "reviews.djs?format=embeddedhtml" RECOMMENDED_URL = "http://origin.api-beta.homedepot.com/ProductServices/v2/products/" \ "recommendation?type=json&key=tRXWvUBGuAwEzFHScjLw9ktZ0Bw7a335" product_filter = [] def __init__(self, *args, **kwargs): # All this is to set the site_name since we have several # allowed_domains. self.br = BuyerReviewsBazaarApi() super(HomedepotProductsSpider, self).__init__(site_name=self.allowed_domains[0], *args, **kwargs) def _parse_single_product(self, response): return self.parse_product(response) @staticmethod def _parse_no_longer_available(response): message = response.xpath( '//div[@class="error" and ' 'contains(., "The product you are trying to view is not currently available.")]' ) return bool(message) def parse_product(self, response): product = response.meta['product'] product['_subitem'] = True if self._parse_no_longer_available(response): product['no_longer_available'] = True return product else: product['no_longer_available'] = False cond_set( product, 'title', response.xpath( "//h1[contains(@class, 'product-title')]/text()").extract()) brand = response.xpath("//h2[@itemprop='brand']/text()").extract() brand = ["".join(brand).strip()] cond_set(product, 'brand', brand) cond_set( product, 'image_url', response.xpath("//div[@class='product_mainimg']/img/@src |" "//img[@id='mainImage']/@src").extract()) cond_set( product, 'price', response.xpath("//div[@class='pricingReg']" "/span[@itemprop='price']/text()").extract()) reseller_id_regex = "\/(\d+)" reseller_id = re.findall(reseller_id_regex, response.url) reseller_id = reseller_id[0] if reseller_id else None cond_set_value(product, 'reseller_id', reseller_id) if product.get('price', None): if not '$' in product['price']: self.log('Unknown currency at' % response.url) else: product['price'] = Price(price=product['price'].replace( ',', '').replace('$', '').strip(), priceCurrency='USD') if not product.get('price'): price = response.xpath( "//div[@class='pricingReg']" "/span[@itemprop='price']/text() |" "//div[contains(@class, 'pricingReg')]/span[@itemprop='price']" ).re(FLOATING_POINT_RGEX) if price: product["price"] = Price(priceCurrency="USD", price=price[0]) try: product['model'] = response.css( '.product_details.modelNo ::text').extract()[0].replace( 'Model', '').replace('#', '').strip() except IndexError: pass internet_no = response.css('#product_internet_number ::text').extract() if internet_no: internet_no = internet_no[0] upc = is_empty(re.findall("ItemUPC=\'(\d+)\'", response.body)) if upc: product["upc"] = upc upc = response.xpath("//upc/text()").re('\d+') if upc: product["upc"] = upc[0] desc = response.xpath("//div[@id='product_description']" "/div[contains(@class,'main_description')]" "/descendant::*[text()]/text()" "//div[contains(@class, 'main_description')] |" "//div[@id='product_description']").extract() desc = " ".join(l.strip() for l in desc if len(l.strip()) > 0) product['description'] = desc product['locale'] = "en-US" metadata = response.xpath( "//script[contains(text(),'PRODUCT_METADATA_JSON')]" "/text()").re('var PRODUCT_METADATA_JSON = (.*);') skus = [] if metadata: metadata = metadata[0] jsmeta = hjson.loads(metadata) try: skus = [jsmeta["attributeDefinition"]["defaultSku"]] response.meta['skus'] = skus metaname = jsmeta['attributeDefinition']['attributeListing'][ 0]['label'] response.meta['attributelabel'] = metaname except (KeyError, IndexError): self.log("Incomplete data from Javascript.", DEBUG) certona_payload = self._gen_payload(response) if certona_payload: new_meta = response.meta.copy() new_meta['product'] = product new_meta['handle_httpstatus_list'] = [404, 415] new_meta['internet_no'] = internet_no headers = { 'Proxy-Connection': 'keep-alive', 'Content-Type': 'application/json' } return Request( self.RECOMMENDED_URL, callback=self._parse_related_products, headers=headers, body=json.dumps(certona_payload), method="POST", meta=new_meta, priority=1000, ) if internet_no: return Request( url=self.REVIEWS_URL % internet_no, callback=self.parse_buyer_reviews, meta={"product": product}, dont_filter=True, ) return self._gen_variants_requests(response, product, skus, internet_no) def _gen_variants_requests(self, response, product, skus, internet_no): reqs = [] for sku in skus: # if sku: # sku = sku[len(sku)-1] new_product = product.copy() new_meta = response.meta.copy() new_meta['product'] = new_product new_meta['handle_httpstatus_list'] = [404] new_meta['internet_no'] = internet_no url = self.DETAILS_URL % sku reqs.append( Request(url, self._parse_skudetails, meta=new_meta, priority=1000)) if not reqs: return product return reqs def _gen_payload(self, response): """Generates request body. Also maxProducts value can be changed for +\- number of values""" # changed version 4.2x -> 5.3x # appid = response.xpath("//input[@id='certona_appId']/@value").extract() # if not appid: # print "no appid" # return appid = 'homedepot01' critemid = response.xpath( "//input[@id='certona_critemId']/@value").extract() if not critemid: critemid = is_empty( re.findall("\"itemId\"\:\"(\d+)\"", response.body)) if not critemid: return payload = { "appId": appid, "products": critemid, "maxProducts": "16", "certonaSchema": "PIPHorizontal1_rr", "sessionId": "41020192309266", "trackingId": "252187705102752", "storeId": "123", } return payload def _parse_related_products(self, response): product = response.meta['product'] internet_no = response.meta.get('internet_no', None) if response.status in response.meta['handle_httpstatus_list']: # No further pages were found. Check the request payload. return product data = json.loads(response.body_as_unicode()) related_prods = [] for prod in data['schemas'][0]['products']: name = prod['productName'] href = prod['canonicalURL'] related_prods.append( RelatedProduct(name, urlparse.urljoin(product['url'], href))) if related_prods: if 'THE HOME DEPOT RECOMMENDS' in data['schemas'][0]['title']: product['related_products'] = {'recommended': related_prods} else: product['related_products'] = { 'buyers_also_bought': related_prods } skus = response.meta.get('skus', None) if not skus: if internet_no: return Request( url=self.REVIEWS_URL % internet_no, callback=self.parse_buyer_reviews, meta={"product": product}, dont_filter=True, ) return product return self._gen_variants_requests(response, product, skus, internet_no) def _parse_skudetails(self, response): product = response.meta['product'] try: jsdata = json.loads(response.body_as_unicode()) storeskus = jsdata['storeSkus'] price = storeskus['storeSku']['pricing']['originalPrice'] product['price'] = price if product.get('price', None): if not '$' in product['price']: self.log('Unknown currency at' % response.url) else: product['price'] = Price(price=product['price'].replace( ',', '').replace('$', '').strip(), priceCurrency='USD') desc = jsdata['info']['description'] product['description'] = desc url = jsdata['canonicalURL'] url = urlparse.urljoin(product['url'], url) product['url'] = url image = jsdata['inlinePlayerJSON']['IMAGE'][1]['mediaUrl'] product['image_url'] = image attrname = response.meta.get('attributelabel', 'Color/Finish') colornames = jsdata['attributeGroups']['group'][0]['entries'][ 'attribute'] colornames = [ el['value'] for el in colornames if el['name'] == attrname ] if colornames: product['model'] = str(colornames[0]) except (ValueError, KeyError, IndexError): self.log("Failed to parse SKU details.", DEBUG) internet_no = response.meta.get('internet_no', None) if internet_no: return Request( url=self.REVIEWS_URL % internet_no, callback=self.parse_buyer_reviews, meta={"product": product}, dont_filter=True, ) return product def parse_buyer_reviews(self, response): product = response.meta.get("product") brs = self.br.parse_buyer_reviews_per_page(response) self.br.br_count = brs.get('num_of_reviews', None) brs['rating_by_star'] = self.br.get_rating_by_star(response) product['buyer_reviews'] = brs return product def _scrape_total_matches(self, response): totals = response.xpath("//a[@id='all_products']/label" "/text()").re(r'All Products \(([\d,]+)\)') if totals: totals = totals[0] totals = totals.replace(",", "") if is_num(totals): return int(totals) no_matches = response.xpath( "//h1[@class='page-title']/text()").extract() if no_matches: if 'we could not find any' in no_matches[0] or \ 'we found 0 matches for' in no_matches[0]: return 0 total_matches = response.xpath( '//*[contains(@id, "all_products")]//text()').extract() if total_matches: total_matches = ''.join(total_matches) total_matches = ''.join(c for c in total_matches if c.isdigit()) if total_matches and total_matches.isdigit(): return int(total_matches) total_matches = response.xpath('//div[@id="allProdCount"]/text()').re( FLOATING_POINT_RGEX) if total_matches: total_matches = total_matches[0] total_matches = total_matches.replace(',', '') if total_matches.isdigit(): return int(total_matches) return def _scrape_product_links(self, response): links = response.xpath( "//div[contains(@class,'product') " "and contains(@class,'plp-grid')]" "//descendant::a[contains(@class, 'item_description')]/@href | " "//div[contains(@class, 'description')]/a[@data-pod-type='pr']/@href" ).extract() if not links: self.log("Found no product links.", DEBUG) for link in links: if link in self.product_filter: continue self.product_filter.append(link) yield link, SiteProductItem() def _scrape_next_results_page_link(self, response): next_page = response.xpath( "//div[@class='pagination-wrapper']/ul/li/span" "/a[@title='Next']/@href |" "//div[contains(@class, 'pagination')]/ul/li/span" "/a[@class='icon-next']/@href |" "//li[contains(@class, 'hd-pagination__item')]" "/a[contains(@class, 'pagination__link') and @title='Next']/@href" ).extract() if next_page: return urlparse.urljoin(response.url, next_page[0])
class RiteAidProductsSpider(ProductsSpider): name = 'riteaid_products' allowed_domains = ['riteaid.com'] SEARCH_URL = "https://shop.riteaid.com/catalogsearch/result/"\ "?limit=72&q={search_term}" _REVIEWS_URL='http://api.bazaarvoice.com/data/reviews.json?apiversion=5.5&passkey=tezax0lg4cxakub5hhurfey5o&' \ 'Filter=ProductId:{sku}&Include=Products&Stats=Reviews' def __init__(self, *args, **kwargs): self.br = BuyerReviewsBazaarApi(called_class=self) super(RiteAidProductsSpider, self).__init__( site_name=self.allowed_domains[0], *args, **kwargs) def _total_matches_from_html(self, response): total = response.xpath( '(//*[@class="pager"]//*[@class="amount"]' '/text())[1]').re('of (\d+)') return int(total[0]) if total else 0 def _scrape_results_per_page(self, response): results_per_page = response.xpath( '//*[@class="limiter"]//option[@selected]/text()').re('\d+') return int(results_per_page[0]) if results_per_page else 0 def _scrape_next_results_page_link(self, response): link = response.xpath('//a[@title="Next"]/@href').extract() return link[0] if link else None def _scrape_product_links(self, response): item_urls = response.xpath( '//*[@class="product-name"]/a/@href').extract() for item_url in item_urls: yield item_url, SiteProductItem() def _parse_single_product(self, response): return self.parse_product(response) def _parse_title(self, response): title = response.xpath('//*[@itemprop="name"]/text()').extract() return title[0] if title else None def _parse_category(self, response): categories = response.xpath( '(//a[@property="v:title"]/text())[position()>1]').extract() return categories[-1] if categories else None def _parse_price(self, response): price = response.xpath('//*[@itemprop="price"]/text()').re('[\d\.]+') currency = response.xpath( '//*[@itemprop="priceCurrency"]/@content').re('\w{2,3}') or ['USD'] if not price: return None return Price(price=price[0], priceCurrency=currency[0]) def _parse_image_url(self, response): image_url = response.xpath('//*[@itemprop="image"]/@src').extract() return image_url[0] if image_url else None def _parse_variants(self, response): return None def _parse_is_out_of_stock(self, response): status = response.xpath( '//*[@itemprop="availability" ' 'and not(@href="http://schema.org/InStock")]') return bool(status) def _parse_description(self, response): description = response.xpath( '(//*[@id="collateral-tabs"]//*[@class="tab-container"])[1]' '//*[self::p or self::ul or self::table] | ' '(//*[@id="collateral-tabs"]//*[@class="tab-container"])[1]' '//*[@class="std"]/text()').extract() return ''.join(description).strip() if description else None def _parse_buyer_reviews(self, response): meta = response.meta.copy() product = response.meta['product'] reqs = meta.get('reqs', []) product['buyer_reviews'] = self.br.parse_buyer_reviews_per_page(response) if reqs: return self.send_next_request(reqs, response) else: return product def _parse_buyer_reviews(self, response): contents = response.body_as_unicode() meta = response.meta.copy() product = response.meta['product'] reqs = meta.get('reqs', []) buyer_reviews = {} sku = product.get('sku') if not product.get('buyer_reviews'): contents = json.loads(contents) incl = contents.get('Includes') brs = incl.get('Products').get(sku) if incl else None if brs: by_star = {} for d in brs['ReviewStatistics']['RatingDistribution']: by_star[str(d['RatingValue'])] = d['Count'] for sc in range(1, 6): if str(sc) not in by_star: by_star[str(sc)] = 0 buyer_reviews['rating_by_star'] = by_star review_count = brs['ReviewStatistics']['TotalReviewCount'] if review_count == 0: product['buyer_reviews'] = ZERO_REVIEWS_VALUE return product buyer_reviews['num_of_reviews'] = review_count average_review = brs['ReviewStatistics']['AverageOverallRating'] average_review = float(format(average_review, '.2f')) buyer_reviews['average_rating'] = average_review product['buyer_reviews'] = BuyerReviews(**buyer_reviews) else: product['buyer_reviews'] = ZERO_REVIEWS_VALUE if not product.get('buyer_reviews'): product['buyer_reviews'] = ZERO_REVIEWS_VALUE if reqs: return self.send_next_request(reqs, response) return product def _parse_last_buyer_date(self, response): last_review_date = response.xpath( '//*[contains(@class,"box-reviews")]' '//*[@class="date"]/text()').re('Posted on (.*)\)') return last_review_date[0] if last_review_date else None def _parse_sku(self, response): sku = response.xpath('.//*[@itemprop="sku"]/@content').extract() return sku[0] if sku else None def _parse_brand(self, response, title): brand = response.xpath('.//*[contains(text(), "Shop all")]/text()').re(r'Shop\sall\s+(\S+)\s?') brand = brand[0].strip() if brand else None if not brand: brand = guess_brand_from_first_words(title) return brand def parse_product(self, response): reqs = [] meta = response.meta.copy() product = meta['product'] # Set locale product['locale'] = 'en_US' # Parse title title = self._parse_title(response) cond_set_value(product, 'title', title, conv=string.strip) # Parse category category = self._parse_category(response) cond_set_value(product, 'category', category) # Parse description description = self._parse_description(response) cond_set_value(product, 'description', description) # Parse price price = self._parse_price(response) cond_set_value(product, 'price', price) # Parse sku sku = self._parse_sku(response) cond_set_value(product, 'sku', sku) # Parse reseller_id cond_set_value(product, 'reseller_id', reseller_id) # Parse image url image_url = self._parse_image_url(response) cond_set_value(product, 'image_url', image_url) # Parse variants variants = self._parse_variants(response) cond_set_value(product, 'variants', variants) # Parse stock status out_of_stock = self._parse_is_out_of_stock(response) cond_set_value(product, 'is_out_of_stock', out_of_stock) # Parse brand brand = self._parse_brand(response, product.get('title')) cond_set_value(product, 'brand', brand) # Parse last buyer review date last_buyer_date = self._parse_last_buyer_date(response) cond_set_value(product, 'last_buyer_review_date', last_buyer_date) # Parse reviews reqs.append( Request( url=self._REVIEWS_URL.format(sku=product['sku']), dont_filter=True, callback=self._parse_buyer_reviews, meta=meta )) if reqs: return self.send_next_request(reqs, response) return product def send_next_request(self, reqs, response): """ Helps to handle several requests """ req = reqs.pop(0) new_meta = response.meta.copy() if reqs: new_meta["reqs"] = reqs return req.replace(meta=new_meta)
class TopshopProductsSpider(ProductsSpider): name = 'topshop_products' allowed_domains = ['topshop.com'] SEARCH_URL = "http://us.topshop.com/webapp/wcs/stores/servlet/CatalogNavigationSearchResultCmd?" \ "langId=-1&storeId=13052&catalogId=33060&Dy=1&Nty=1&beginIndex=1&pageNum=1&Ntt={search_term}" _REVIEWS_URL = 'http://topshop.ugc.bazaarvoice.com/6025-en_us/{sku}/reviews.djs?format=embeddedhtml' def __init__(self, *args, **kwargs): self.br = BuyerReviewsBazaarApi(called_class=self) super(TopshopProductsSpider, self).__init__(site_name=self.allowed_domains[0], *args, **kwargs) def _total_matches_from_html(self, response): total = response.xpath('(//*[@class="pager"]//*[@class="amount"]' '/text())[1]').re('of (\d+)') return int(total[0]) if total else 0 def _scrape_results_per_page(self, response): results_per_page = response.xpath( '//*[@class="limiter"]//option[@selected]/text()').re('\d+') return int(results_per_page[0]) if results_per_page else 0 def _scrape_next_results_page_link(self, response): link = response.xpath('//a[@title="Next"]/@href').extract() return link[0] if link else None def _scrape_product_links(self, response): item_urls = response.xpath( '//*[@class="product-name"]/a/@href').extract() for item_url in item_urls: yield item_url, SiteProductItem() def _parse_single_product(self, response): return self.parse_product(response) def _parse_title(self, response): title = response.xpath('//*[@itemprop="name"]/text()').extract() return title[0] if title else None def _parse_category(self, response): categories = response.xpath( '//*[@id="nav_breadcrumb"]//li//a//span//text()').extract() return categories[-1] if categories else None def _parse_price(self, response): price = response.xpath( '//div[contains(@class,"product_details")]' '//div[contains(@class,"product_prices")]//span//text()').extract( ) if len(price) > 1: price = price[1] if "$" in price: currency = 'USD' else: currency = '' price = re.findall(r'[\d\.]+', price) if len(price) == 0: return None return Price(price=price[0], priceCurrency=currency) def _parse_image_url(self, response): image_url = response.xpath( '//ul[contains(@class,"product_hero__wrapper")]' '//a[contains(@class,"hero_image_link")]//img/@src').extract() return image_url[0] if image_url else None def _parse_variants(self, response): return None def _parse_is_out_of_stock(self, response): status = response.xpath('//*[@itemprop="availability" ' 'and not(@href="http://schema.org/InStock")]') return bool(status) def _parse_description(self, response): description = response.xpath( '//div[@id="productInfo"]//p//text()').extract() return ''.join(description).strip() if description else None def _parse_buyer_reviews(self, response): meta = response.meta.copy() product = response.meta['product'] reqs = meta.get('reqs', []) product['buyer_reviews'] = self.br.parse_buyer_reviews_per_page( response) if reqs: return self.send_next_request(reqs, response) else: return product def clear_text(self, str_result): return str_result.replace("\t", "").replace("\n", "").replace("\r", "").strip() def _parse_buyer_reviews(self, response): product = response.meta['product'] reqs = response.meta.get('reqs', []) content = re.search('BVRRRatingSummarySourceID":"(.+?)\},', response._body).group(1).replace('\\"', '"') content = content.replace("\\/", "/") review_html = html.fromstring(content) arr = review_html.xpath( '//div[contains(@class,"BVRRQuickTakeSection")]' '//div[contains(@class,"BVRRRatingOverall")]' '//img[contains(@class,"BVImgOrSprite")]/@title') if len(arr) > 0: average_rating = float(arr[0].strip().split(" ")[0]) else: average_rating = 0.0 arr = review_html.xpath( '//div[contains(@class,"BVRRReviewDisplayStyle5")]' '//div[contains(@class,"BVRRReviewDisplayStyle5Header")]' '//span[@itemprop="ratingValue"]//text()') num_of_reviews = len(arr) review_list = [[5 - i, arr.count(str(5 - i))] for i in range(5)] if review_list: # average score sum = 0 cnt = 0 for i, review in review_list: sum += review * i cnt += review # average_rating = float(sum)/cnt # number of reviews num_of_reviews = 0 for i, review in review_list: num_of_reviews += review else: pass rating_by_star = {1: 0, 2: 0, 3: 0, 4: 0, 5: 0} for i, review in review_list: rating_by_star[i] = review if average_rating and num_of_reviews: product["buyer_reviews"] = BuyerReviews( num_of_reviews=int(num_of_reviews), average_rating=float(average_rating), rating_by_star=rating_by_star, ) else: product["buyer_reviews"] = ZERO_REVIEWS_VALUE if reqs: return self.send_next_request(reqs, response) return product def _parse_last_buyer_date(self, response): last_review_date = response.xpath( '//*[contains(@class,"box-reviews")]' '//*[@class="date"]/text()').re('Posted on (.*)\)') return last_review_date[0] if last_review_date else None def _parse_sku(self, response): sku = response.xpath( '//div[@id="productInfo"]//li[contains(@class,"product_code")]//span//text()' ).extract() return sku[0] if sku else None def _parse_brand(self, response, title): brand = response.xpath('.//*[contains(text(), "Shop all")]/text()').re( r'Shop\sall\s+(\S+)\s?') brand = brand[0].strip() if brand else None if not brand: try: brand = guess_brand_from_first_words(title) except: brand = None return brand def parse_product(self, response): reqs = [] meta = response.meta.copy() product = meta['product'] # Set locale product['locale'] = 'en_US' # Parse title title = self._parse_title(response) cond_set_value(product, 'title', title, conv=string.strip) # Parse category category = self._parse_category(response) cond_set_value(product, 'category', category) # Parse description description = self._parse_description(response) cond_set_value(product, 'description', description) # Parse price price = self._parse_price(response) cond_set_value(product, 'price', price) # Parse sku sku = self._parse_sku(response) cond_set_value(product, 'sku', sku) # Parse image url image_url = self._parse_image_url(response) cond_set_value(product, 'image_url', image_url) # Parse variants variants = self._parse_variants(response) cond_set_value(product, 'variants', variants) # Parse stock status out_of_stock = self._parse_is_out_of_stock(response) cond_set_value(product, 'is_out_of_stock', out_of_stock) # Parse brand brand = self._parse_brand(response, product.get('title')) cond_set_value(product, 'brand', brand) # Parse last buyer review date last_buyer_date = self._parse_last_buyer_date(response) cond_set_value(product, 'last_buyer_review_date', last_buyer_date) # Parse reviews reqs.append( Request(url=self._REVIEWS_URL.format(sku=product['sku']), dont_filter=True, callback=self._parse_buyer_reviews, meta=meta)) if reqs: return self.send_next_request(reqs, response) return product def send_next_request(self, reqs, response): """ Helps to handle several requests """ req = reqs.pop(0) new_meta = response.meta.copy() if reqs: new_meta["reqs"] = reqs return req.replace(meta=new_meta)
class PepboysProductsSpider(ProductsSpider): name = 'pepboys_products' allowed_domains = ['pepboys.com'] SEARCH_URL = "http://www.pepboys.com/s?query={search_term}" BUYER_REVIEWS_URL = ("https://pepboys.ugc.bazaarvoice.com/8514-en_us" "/{product_id}/reviews.djs?format=embeddedhtml") def __init__(self, *args, **kwargs): super(PepboysProductsSpider, self).__init__(*args, **kwargs) self.br = BuyerReviewsBazaarApi(called_class=self) def _total_matches_from_html(self, response): total = response.css('.resultCount::text').re('of (\d+) Result') return int(total[0].replace(',', '')) if total else 0 def _scrape_results_per_page(self, response): return 39 def _scrape_next_results_page_link(self, response): link = response.xpath('//a[@class="next"]/@href').extract() return link[0] if link else None def _scrape_product_links(self, response): item_urls = response.xpath( '//*[@class="product"]/a[1]/@href').extract() for item_url in item_urls: yield item_url, SiteProductItem() def _parse_single_product(self, response): return self.parse_product(response) def _parse_title(self, response): title = response.xpath( '//h4[contains(@class,"margin-top-none")]//text()').extract() title = [r.strip() for r in title if len(r.strip()) > 0] title = "".join(title) return title.strip() if title else None def _parse_categories(self, response): categories = response.xpath( '//*[@class="breadcrumb"]//li/a/text()').extract() return categories if categories else None def _parse_category(self, response): categories = self._parse_categories(response) return categories[-1] if categories else None def _parse_price(self, response): try: price = response.xpath( '//div[contains(@class,"subtotal")]//span[@class="price"]//text()' ).extract()[0].strip() price = re.findall(r'[\d\.]+', price) except: return None if not price: return None return Price(price=price[0], priceCurrency='USD') def _parse_image_url(self, response): image_url = response.xpath( '//img[contains(@class,"tdTireDetailImg")]/@src').extract() return image_url[0] if image_url else None def _parse_brand(self, response): brand = 'Pepboys' return brand.strip() if brand else None def _parse_sku(self, response): sku = response.xpath( '//div[contains(@class,"j-results-item-container")]/@data-sku' ).extract() return sku[0] if sku else None def _parse_variants(self, response): return None def _parse_is_out_of_stock(self, response): status = response.xpath( '//*[@id="availability"]/span[text()="In Stock"]') return not bool(status) def _parse_shipping_included(self, response): shipping_text = ''.join( response.xpath('//span[@class="free-shipping"]//text()').extract()) return shipping_text == ' & FREE Shipping' def _parse_description(self, response): description = response.xpath( '//div[contains(@class,"tdContentDesc")]').extract() return ''.join(description).strip() if description else None def _parse_buyer_reviews(self, response): str_review = response.xpath( '//div[@class="tsrSeeReviews"]//text()').extract() if str_review: num_of_reviews = re.findall(r'[\d]+', str_review) if len(num_of_reviews) > 0: num_of_reviews = num_of_reviews[0] buyer_reviews = { 'num_of_reviews': int(num_of_reviews), 'average_rating': float(average_rating), 'rating_by_star': rating_by_star } product = response.meta['product'] buyer_reviews = self.br.parse_buyer_reviews_per_page(response) product['buyer_reviews'] = buyer_reviews yield product def send_next_request(self, reqs, response): """ Helps to handle several requests """ req = reqs.pop(0) new_meta = response.meta.copy() if reqs: new_meta["reqs"] = reqs return req.replace(meta=new_meta) def parse_product(self, response): reqs = [] product = response.meta['product'] response.meta['product_response'] = response # Set locale product['locale'] = 'en_US' # Parse title title = self._parse_title(response) cond_set_value(product, 'title', title, conv=string.strip) # Parse category category = self._parse_category(response) cond_set_value(product, 'category', category) # Parse categories categories = self._parse_categories(response) cond_set_value(product, 'categories', categories) # Parse description description = self._parse_description(response) cond_set_value(product, 'description', description) # Parse price price = self._parse_price(response) cond_set_value(product, 'price', price) # Parse image url image_url = self._parse_image_url(response) cond_set_value(product, 'image_url', image_url) # Parse variants variants = self._parse_variants(response) cond_set_value(product, 'variants', variants) # Parse stock status out_of_stock = self._parse_is_out_of_stock(response) cond_set_value(product, 'is_out_of_stock', out_of_stock) # Sku sku = self._parse_sku(response) cond_set_value(product, 'sku', sku) # Brand brand = self._parse_brand(response) cond_set_value(product, 'brand', brand) # Shipping included shipping_included = self._parse_shipping_included(response) cond_set_value(product, 'shipping_included', shipping_included) # Custom reviews if sku: # Parse buyer reviews reqs.append( Request(url=self.BUYER_REVIEWS_URL.format(product_id=sku), dont_filter=True, callback=self.br.parse_buyer_reviews)) if reqs: return self.send_next_request(reqs, response) return product def _get_products(self, response): remaining = response.meta['remaining'] search_term = response.meta['search_term'] prods_per_page = response.meta.get('products_per_page') total_matches = response.meta.get('total_matches') scraped_results_per_page = response.meta.get( 'scraped_results_per_page') prods = self._scrape_product_links(response) if prods_per_page is None: # Materialize prods to get its size. prods = list(prods) prods_per_page = len(prods) response.meta['products_per_page'] = prods_per_page if scraped_results_per_page is None: scraped_results_per_page = self._scrape_results_per_page(response) if scraped_results_per_page: self.log( "Found %s products at the first page" % scraped_results_per_page, INFO) else: scraped_results_per_page = prods_per_page if hasattr(self, 'is_nothing_found'): if not self.is_nothing_found(response): self.log( "Failed to scrape number of products per page", ERROR) response.meta[ 'scraped_results_per_page'] = scraped_results_per_page if total_matches is None: total_matches = self._scrape_total_matches(response) if total_matches is not None: response.meta['total_matches'] = total_matches self.log("Found %d total matches." % total_matches, INFO) else: if hasattr(self, 'is_nothing_found'): if not self.is_nothing_found(response): self.log( "Failed to parse total matches for %s" % response.url, ERROR) if total_matches and not prods_per_page: # Parsing the page failed. Give up. self.log("Failed to get products for %s" % response.url, ERROR) return for i, (prod_url, prod_item) in enumerate(islice(prods, 0, remaining)): # Initialize the product as much as possible. prod_item['site'] = self.site_name prod_item['search_term'] = search_term prod_item['total_matches'] = total_matches prod_item['results_per_page'] = prods_per_page prod_item['scraped_results_per_page'] = scraped_results_per_page # The ranking is the position in this page plus the number of # products from other pages. prod_item['ranking'] = (i + 1) + (self.quantity - remaining) if self.user_agent_key not in ["desktop", "default"]: prod_item['is_mobile_agent'] = True if prod_url is None: # The product is complete, no need for another request. yield prod_item elif isinstance(prod_url, Request): cond_set_value(prod_item, 'url', prod_url.url) # Tentative. yield prod_url else: # Another request is necessary to complete the product. url = urlparse.urljoin(response.url, prod_url) cond_set_value(prod_item, 'url', url) # Tentative. yield Request( url, callback=self.parse_product, meta={'product': prod_item}, # Remove Referer field on searchs to make the # website display the breadcrumbs headers={'referer': ''}, )
class NikeProductSpider(BaseProductsSpider): name = 'nike_products' allowed_domains = ["nike.com"] SEARCH_URL = "http://nike.com/#{search_term}" REVIEW_URL = "http://nike.ugc.bazaarvoice.com/9191-en_us/{product_model}" \ "/reviews.djs?format=embeddedhtml" #handle_httpstatus_list = [404, 403, 429] use_proxies = False # we'll be using Crawlera instead def __init__(self, sort_mode=None, *args, **kwargs): from scrapy.conf import settings settings.overrides['DEPTH_PRIORITY'] = 1 settings.overrides[ 'SCHEDULER_DISK_QUEUE'] = 'scrapy.squeue.PickleFifoDiskQueue' settings.overrides[ 'SCHEDULER_MEMORY_QUEUE'] = 'scrapy.squeue.FifoMemoryQueue' settings.overrides['CRAWLERA_ENABLED'] = True self.quantity = kwargs.get('quantity', 1000) # default is 1000 self.proxy = 'content.crawlera.com:8010' self.proxy_type = 'http' #self.user_agent = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:32.0) Gecko/20100101 Firefox/32.0' self.user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A' self.br = BuyerReviewsBazaarApi(called_class=self) super(NikeProductSpider, self).__init__(site_name=self.allowed_domains[0], *args, **kwargs) @staticmethod def _get_antiban_headers(): return { 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:32.0) Gecko/20100101 Firefox/32.0', 'Connection': 'keep-alive', 'Accept-Language': 'en-US,en;q=0.5', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Encoding': 'gzip, deflate' } def start_requests(self): for st in self.searchterms: yield Request(self.url_formatter.format( self.SEARCH_URL, search_term=st.encode('utf-8'), ), meta={ 'search_term': st, 'remaining': self.quantity }, headers=self._get_antiban_headers()) if self.product_url: prod = SiteProductItem() prod['is_single_result'] = True meta = {} meta['is_product_page'] = True meta['product'] = prod yield Request(self.product_url, self._parse_single_product, meta=meta, headers=self._get_antiban_headers()) def _init_firefox(self): from selenium import webdriver from selenium.webdriver.remote.remote_connection import RemoteConnection RemoteConnection.set_timeout(30) profile = webdriver.FirefoxProfile() profile.set_preference("general.useragent.override", self.user_agent) profile.set_preference('intl.accept_languages', 'en-US') profile.set_preference("network.proxy.type", 1) # manual proxy configuration profile.set_preference('permissions.default.image', 2) if self.proxy: profile.set_preference("network.http.phishy-userpass-length", 255) if 'socks' in self.proxy_type: profile.set_preference("network.proxy.socks", self.proxy.split(':')[0]) profile.set_preference("network.proxy.socks_port", int(self.proxy.split(':')[1])) else: profile.set_preference("network.proxy.http", self.proxy.split(':')[0]) profile.set_preference("network.proxy.http_port", int(self.proxy.split(':')[1])) profile.update_preferences() driver = webdriver.Firefox(profile) driver.set_window_size(1280, 1024) driver.set_page_load_timeout(60) driver.set_script_timeout(60) return driver def _parse_single_product(self, response): return self.parse_product(response) def _get_product_links_from_serp(self, driver): results = [] links = driver.find_elements_by_xpath( '//*[contains(@class, "grid-item-image")]' '//a[contains(@href, "/pd/") or contains(@href, "/product/")]') for l in links: href = l.get_attribute('href') if href: if not href.startswith('http'): href = urlparse.urljoin( 'http://' + self.allowed_domains[0], href) results.append(href) return results def _is_product_page(self, response): return 'is_product_page' in response.meta @staticmethod def _get_proxy_ip(driver): driver.get('http://icanhazip.com') ip = re.search('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', driver.page_source) if ip: ip = ip.group(1) return ip @staticmethod def _auth_firefox_proxy(driver): driver.set_page_load_timeout(10) try: driver.get('http://icanhazip.com') except: from selenium.webdriver.common.alert import Alert time.sleep(3) alert = Alert(driver) time.sleep(3) #alert.authenticate(CRAWLERA_APIKEY, '') alert.send_keys(CRAWLERA_APIKEY + '\n') alert.accept() #alert.send_keys('\t') #alert.send_keys('\n') #import pdb; pdb.set_trace() driver.set_page_load_timeout(30) @staticmethod def last_five_digits_the_same(lst): if len(lst) < 6: return return lst[-1] == lst[-2] == lst[-3] == lst[-4] == lst[-5] def _reliable_get(self, driver, url, max_attempts=40, check_element='title'): """ Acts like driver.get() but with failsafety """ driver.set_page_load_timeout(30) driver.set_script_timeout(30) for i in range(max_attempts): try: driver.get(url) if driver.find_elements_by_xpath('//%s' % check_element): return driver except: self.log('_reliable_get error #%i while getting url %s' % (i, url)) self.log('_reliable_get failed to get url %s' % url, ERROR) def parse(self, response): if not self._is_product_page(response): display = Display(visible=0, size=(1024, 768)) display.start() product_links = [] # scrape "quantity" products driver = self._init_firefox() self._auth_firefox_proxy(driver) if self.proxy: ip_via_proxy = NikeProductSpider._get_proxy_ip(driver) print 'IP via proxy:', ip_via_proxy self.log('IP via proxy: %s' % ip_via_proxy) try: self._reliable_get(driver, 'http://store.nike.com/us/en_us') except Exception as e: print(str(e)) self.log(str(e), WARNING) driver.find_element_by_name('searchList').send_keys( self.searchterms[0] + '\n') time.sleep(6) # let AJAX finish new_meta = response.meta.copy() # get all products we need (scroll down) collected_products_len = [] num_exceptions = 0 while 1: try: driver.execute_script("scrollTo(0,50000)") time.sleep(10) product_links = self._get_product_links_from_serp(driver) collected_products_len.append(len(product_links)) print 'Collected %i product links' % len(product_links) self.log('Collected %i product links' % len(product_links)) if len(product_links) > self.quantity: break if self.last_five_digits_the_same(collected_products_len): break # last five iterations collected equal num of products except Exception as e: print str(e) self.log('Exception while scrolling page: %s' % str(e), WARNING) num_exceptions += 1 if num_exceptions > 10: self.log('Maximum number of exceptions reached', ERROR) driver.quit() display.stop() return for i in xrange(10): time.sleep(3) try: selenium_cookies = driver.get_cookies() break except Exception as e: print('Exception while loading cookies %s attempt %i' % (str(e), i)) self.log('Exception while loading cookies %s attempt %i' % (str(e), i)) try: driver.quit() display.stop() except: pass #driver.save_screenshot('/tmp/1.png') new_meta['is_product_page'] = True new_meta['proxy'] = self.proxy for i, product_link in enumerate(product_links): new_meta['_ranking'] = i + 1 yield Request(product_link, meta=new_meta, callback=self.parse_product, headers=self._get_antiban_headers()) #cookies=selenium_cookies) def parse_product(self, response): meta = response.meta.copy() product = meta.get('product', SiteProductItem()) product['_subitem'] = True _ranking = response.meta.get('_ranking', None) product['ranking'] = _ranking product['url'] = response.url product['search_term'] = response.meta.get('search_term', None) # product data in json js_data = self.parse_data(response) # product id product_id = self.parse_product_id(response, js_data) product_color = self.parse_product_color(response, js_data) product_price = 0 # Parse product_id title = self.parse_title(response, js_data) cond_set_value(product, 'title', title) if not product.get('title', None): return # Parse locate locale = 'en_US' cond_set_value(product, 'locale', locale) # Parse model product_model = self.parse_product_model(response) cond_set_value(product, 'model', product_model) # Parse image image = self.parse_image(response, js_data) cond_set_value(product, 'image_url', image) # Parse reseller_id reseller_id = self.parse_reseller_id(response) cond_set_value(product, "reseller_id", reseller_id) # Parse brand # brand = self.parse_brand(response) # cond_set_value(product, 'brand', brand) # Parse upc # upc = self.parse_upc(response) # cond_set_value(product, 'upc', upc) # Parse sku sku = self.parse_sku(response) cond_set_value(product, 'sku', sku) # Parse description description = self.parse_description(response) cond_set(product, 'description', description) # Parse price price = self.parse_price(response, js_data) cond_set_value(product, 'price', price) # Parse variants nv = NikeVariants() nv.setupSC(response) try: product['variants'] = nv._variants() except: # "/product/" urls that are non-standard and not supported (yet)? pass meta['product'] = product # parse buyer reviews yield Request(url=self.REVIEW_URL.format(product_model=product_model), dont_filter=True, callback=self.parse_buyer_reviews, meta=meta) yield product def parse_reseller_id(self, response): regex = "\/pid-(\d+)" reseller_id = re.findall(regex, response.url) reseller_id = reseller_id[0] if reseller_id else None return reseller_id def parse_count_reviews(self, response): count_review = response.xpath( '//meta[contains(@itemprop, "reviewCount")]/@content').extract() if count_review: return int(count_review[0]) else: return 0 def parse_data(self, response): script_data = response.xpath( '//script[contains(@id, "product-data")]/text()').extract() try: js_data = json.loads(script_data[0]) return js_data except: return def parse_image(self, response, js_data): if js_data: try: image = js_data['imagesHeroLarge'][0] return image except: return def parse_description(self, response): # js_data['content'] desc = response.xpath( '//div[contains(@class, "pi-pdpmainbody")]').extract() return desc def parse_sku(self, response): skuid = response.xpath( '//span[contains(@class, "exp-style-color")]/text()').extract() if skuid: return skuid[0].replace('Style: ', '') def parse_price(self, response, js_data): if js_data: try: currency = js_data['crossSellConfiguration']['currency'] except KeyError: currency = "USD" try: price = js_data['rawPrice'] self.product_price = price except KeyError: price = 0.00 if price and currency: price = Price(price=price, priceCurrency=currency) else: price_og = re.search( '<meta property="og:price:amount" content="([\d\.]+)" />', response.body_as_unicode()) if price_og: return Price(price=float(price_og.group(1)), priceCurrency="USD") price = Price(price=0.00, priceCurrency="USD") return price def _scrape_total_matches(self, response): totals = response.css('.productCount ::text').extract() if totals: totals = totals[0].replace(',', '').replace('.', '').strip() if totals.isdigit(): if not self.TOTAL_MATCHES: self.TOTAL_MATCHES = int(totals) return int(totals) def _scrape_product_links(self, response): for link in response.xpath( '//li[contains(@class, "product-tile")]' '//a[contains(@rel, "product")]/@href').extract(): yield link, SiteProductItem() def parse_product_id(self, response, js_data): if js_data: try: product_id = js_data['productId'] return product_id except: return def parse_product_model(self, response): model = response.xpath( '//div[contains(@class, "hero-product-style-color-info")]/@data-stylenumber' ).extract() return model[0] if model else None def parse_product_color(self, response, js_data): if js_data: try: product_color = js_data['colorDescription'] return product_color except: return def parse_title(self, response, js_data): if js_data: try: title = js_data['productTitle'] return title except: return def parse_buyer_reviews(self, response): buyer_reviews_per_page = self.br.parse_buyer_reviews_per_page(response) product = response.meta['product'] product['buyer_reviews'] = BuyerReviews(**buyer_reviews_per_page) yield product