def parse_color_item(self, response): sel = Selector(response) index = response.meta['index'] color_urls = response.meta['color_urls'] baseItem = response.meta['baseItem'] colorItem = Color() colorItem['type'] = 'color' colorItem['show_product_id'] = baseItem['show_product_id'] colorItem['from_site'] = self.name colorItem['name'] = color_urls[index]['color_name'] colorItem['cover'] = color_urls[index]['color_cover'] images = [] imageItem = ImageItem() image_url = sel.xpath('//meta[@property="og:image"]/@content').extract()[0] imageItem['image'] = re.sub(r'wid=\d+&hei=\d+', 'wid=1000&hei=1000', image_url) imageItem['thumbnail'] = re.sub(r'wid=\d+&hei=\d+', 'wid=50&hei=50', image_url) images.append(imageItem) image_url2 = sel.xpath('//div[@id="productSwatch"]/img/@src').extract()[0] imageItem = ImageItem() imageItem['image'] = re.sub(r'wid=\d+&hei=\d+', 'wid=1000&hei=1000', image_url2) imageItem['thumbnail'] = re.sub(r'wid=\d+&hei=\d+', 'wid=50&hei=50', image_url2) images.append(imageItem) colorItem['images'] = images yield colorItem skus = response.meta['skus'] skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = baseItem['show_product_id'] skuItem['from_site'] = self.name skuItem['current_price'] = sel.xpath('//div[@class="prodprice saleprice"]/p/span[@itemprop="price"]/text()').extract()[0] if len(sel.xpath('//div[@class="prodprice saleprice"]/p/span[@class="basePrice"]/text()').extract()) > 0: skuItem['list_price'] = sel.xpath('//div[@class="prodprice saleprice"]/p/span[@class="basePrice"]/text()').extract()[0] else: skuItem['list_price'] = skuItem['current_price'] skuItem['is_outof_stock'] = False skuItem['color'] = color_urls[index]['color_name'] skuItem['size'] = 'one-size' skuItem['id'] = baseItem['show_product_id'] skus.append(skuItem) if index + 1 == len(color_urls): baseItem['skus'] = skus yield baseItem else: yield Request(color_urls[index+1]['url'], callback=self.parse_color_item , meta={'baseItem': baseItem, 'color_urls': color_urls, 'index': index+1, 'skus': skus})
def parse_image(self, response): color_item = response.meta['item'] show_product_id = response.meta['show_product_id'] sel = Selector(response) image_boxs = sel.css('.magicThumbBox .clickStreamTag') images = [] for image_box in image_boxs: imageItem = ImageItem() image = image_box.xpath('@href').extract()[0] thumbnail = image_box.xpath('//img/@src').extract()[0] if re.match(r'^//', image): image = 'https:' + image if re.match(r'^//', thumbnail): thumbnail = 'https:' + thumbnail imageItem['image'] = image imageItem['thumbnail'] = thumbnail images.append(imageItem) if 'cover' not in color_item or 'cover_style' not in color_item: color_item['cover'] = images[0]['thumbnail'] color_item["images"] = images color_item['show_product_id'] = show_product_id yield color_item
def parse_color_sku(self, response): baseItem = response.meta['baseItem'] images_tmp = response.meta['images'] jsonStr = json.loads(response.body) colors = [] skus = [] sizes = [] for col in jsonStr['Colors']: images = [] for img in images_tmp: imageItem = ImageItem() imageItem['thumbnail'] = '%s%s_%s_%s.%s' % ( img['base'], col['Code10'], img['thum_size'], img['index'], img['thum_ext']) imageItem['image'] = '%s%s_%s_%s.%s' % ( img['base'], col['Code10'], img['img_size'], img['index'], img['img_ext']) images.append(imageItem) color = Color() color['type'] = 'color' color['from_site'] = 'thecorner' color['show_product_id'] = baseItem['show_product_id'] color['images'] = images color['name'] = col['Description'] color['cover_style'] = '#' + col['Rgb'] #color['cover_style'] = 'background-color: #%s;' % (col['Rgb']) colors.append(col['Description']) yield color for size in jsonStr['ModelColorSizes']: skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = baseItem['show_product_id'] skuItem['from_site'] = 'thecorner' skuItem['id'] = size['Color']['Description'].encode( "utf-8") + "*" + size['Size']['Description'] skuItem['list_price'] = baseItem['list_price'] skuItem['current_price'] = baseItem['current_price'] skuItem['size'] = size['Size']['Description'] skuItem['color'] = size['Color']['Description'] skuItem['is_outof_stock'] = False skuItem['quantity'] = size['Quantity'] sizes.append(size['Size']['Description']) skus.append(skuItem) baseItem['skus'] = skus baseItem['colors'] = list(set(colors)) baseItem['sizes'] = list(set(sizes)) yield baseItem
def handle_image_item(self, response): item = response.meta['item'] sel = Selector(response) colorIds = response.meta['colorIds'] colorUrls = response.meta['colorUrls'] image_color_dict = {} for colorId in colorIds: images = [] single_color_imgages = re.findall( "[^\.{]*\/zoom_variation_" + colorId + "[^{]*2200\.jpg", response.body) if len(single_color_imgages) == 0: single_color_imgages = re.findall( "[^\.{]*\/zoom_variation_" + colorIds[0] + "[^{]*2200\.jpg", response.body) for i in range(len(single_color_imgages)): imageItem = ImageItem() imageItem[ 'image'] = self.data_url + single_color_imgages[i][1:] tempImage = re.sub('zoom_variation_', 'main_variation_', imageItem['image']) imageItem['thumbnail'] = re.sub('2192x2200', '548x550', tempImage) images.append(imageItem.copy()) image_color_dict[colorId] = images index = 0 yield Request(colorUrls[index], callback=self.handle_color_item, meta={ 'image_color_dict': image_color_dict, 'colorUrls': colorUrls, 'index': index, 'item': item })
def parse_img(self, response): sel = Selector(response) color = response.meta['color'] images_coloum = sel.xpath( '//div[contains(@id, "rl_pdd_cover_slider")]//ul[contains(@class, "box")]//li' ) images = [] for images_col in images_coloum: imageItem = ImageItem() imageItem['image'] = images_col.xpath('./img/@src').extract()[0] imageItem['thumbnail'] = images_col.xpath( './img/@small').extract()[0] images.append(imageItem) color['images'] = images yield color
def handle_parse_item(self, response, baseItem): sel = Selector(response) baseItem['dimensions'] = ['size', 'color'] baseItem['desc'] = sel.xpath( '//div[@id="aDescriptionBody"]').extract()[0] baseItem['brand'] = sel.xpath( '//span[@itemprop="brand"]/text()').extract()[0] skus = [] skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = baseItem['show_product_id'] skuItem['from_site'] = self.name skuItem['current_price'] = baseItem['current_price'] skuItem['list_price'] = baseItem['list_price'] skuItem['is_outof_stock'] = False skuItem['color'] = 'one-color' skuItem['size'] = 'one-size' skuItem['id'] = baseItem['show_product_id'] skus.append(skuItem) imageItem = ImageItem() imageItem['image'] = 'http:' + sel.xpath( '//img[@id="ImageUrl"]/@src').extract()[0] imageItem['thumbnail'] = imageItem['image'] images = [] images.append(imageItem) colorItem = Color() colorItem['type'] = 'color' colorItem['show_product_id'] = baseItem['show_product_id'] colorItem['from_site'] = self.name colorItem['images'] = images colorItem['name'] = 'one-color' yield colorItem baseItem['skus'] = skus yield baseItem
def handle_parse_item(self, response, item): sel = Selector(response) catalogId = sel.xpath( '//input[contains(@id, "catalogId")]/@value').extract()[0] storeId = sel.xpath( '//input[contains(@id, "storeId")]/@value').extract()[0] show_product_id = sel.xpath( '//input[contains(@id, "productId")]/@value').extract()[0] #baseItem = response.meta['baseItem'] baseItem = item baseItem['from_site'] = self.name baseItem['type'] = 'base' baseItem['title'] = sel.xpath( '//div[contains(@class, "pdd_title box")]//h3/text()').extract()[0] baseItem['show_product_id'] = show_product_id baseItem['desc'] = sel.xpath( '//div[contains(@class, "pdd_desc pdd_sub_item box")]').extract( )[0] baseItem['list_price'] = sel.xpath( '//p[contains(@class, "pdd_price")]//span/text()').extract()[0] if sel.xpath('//span[contains(@class, "promo_price")]'): baseItem['current_price'] = sel.xpath( '//span[contains(@class, "promo_price")]/text()').extract()[0] else: baseItem['current_price'] = baseItem['list_price'] colors = [] skus = [] colorNames = [] sizes = {'size': ['onesize']} color_coloum = sel.xpath( '//ul[contains(@class, "pdd_colors_list box")]//li') images_coloum = sel.xpath( '//div[contains(@id, "rl_pdd_cover_slider")]//ul[contains(@class, "box")]//li' ) for colors_col in color_coloum: color = Color() color['type'] = 'color' color['from_site'] = self.name color['show_product_id'] = show_product_id color['cover'] = colors_col.xpath('./a/img/@src').extract()[0] color['name'] = colors_col.xpath('./a/img/@alt').extract()[0] colorNames.append(color['name']) if colors_col.xpath( './a[contains(@class, "pdd_color pdd_color_picked")]//span/@onclick' ): images = [] for images_col in images_coloum: imageItem = ImageItem() imageItem['image'] = images_col.xpath( './img/@src').extract()[0] imageItem['thumbnail'] = images_col.xpath( './img/@small').extract()[0] images.append(imageItem) color['images'] = images yield color else: #print re.findall(r'\(.*?\)', clickParam) clickParam = colors_col.xpath( './a[contains(@class, "pdd_color")]//span/@onclick' ).extract()[0] clickParams = re.findall(r"'(.*)'", clickParam)[0].split(',') MFPARTNUMBER = clickParams[5].replace("'", "") imgUrl = '%s/webapp/wcs/stores/servlet/ProductDetailFullImageView?catalogId=%s&langId=-1&storeId=%s&MFPARTNUMBER=%s' % ( self.base_url, catalogId, storeId, MFPARTNUMBER) yield Request(imgUrl.encode('UTF-8'), meta={'color': color}, callback=self.parse_img) ''' quantityUrl = 'http://www.ralphlauren.asia/webapp/wcs/stores/servlet/ProductDetailQuantityView?catalogId=12551&langId=-1&storeId=12151' formdata = { 'SKUId': str(clickParam[2]), 'objectId':'', 'requesttype':'ajax' } yield FormRequest(url=sizeUrl, formdata=formdata, callback=self.parse_quantity, meta={ '': '' } ) sizeUrl = self.base_url + '/webapp/wcs/stores/servlet/ProductDetailSizeSelectView?catalogId=12551&langId=-1&storeId=12151' formdata = { 'Id': str(clickParam[1]), 'SKUId': str(clickParam[2]), 'Color': 'Lime', 'ColorId': str(clickParam[5]), 'Size': '', 'InItemSppSplitChar':'@@', 'objectId':'', 'requesttype':'ajax' } yield FormRequest(url=sizeUrl, formdata=formdata, callback=self.parse_size, meta={ 'color': color } ) ''' ### sku_coloum = sel.xpath( '//select[contains(@id, "rl_pdd_size")]//option ') for sku_col in sku_coloum: skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = show_product_id skuItem['from_site'] = self.name skuItem['list_price'] = baseItem['list_price'] skuItem['current_price'] = baseItem['current_price'] skuItem['is_outof_stock'] = False skuItem['id'] = sel.xpath( '//input[contains(@id, "selectSKUId")]/@value').extract()[0] skuItem['color'] = sel.xpath( '//span[contains(@class, "pdd_current_color")]/text()' ).extract()[0] #skuItem['size'] = sku_col.xpath('./option/text()').extract()[0] skuItem['quantity'] = sel.xpath( '//select[contains(@id, "rl_pdd_qty")]//option/@value' ).extract()[0] skus.append(skuItem) baseItem['colors'] = colorNames baseItem['sizes'] = sizes baseItem['skus'] = skus baseItem['dimensions'] = "" baseItem['brand'] = 'ralph lauren' baseItem['category'] = sel.xpath( '//div[contains(@class, "bread bread_bar")]//a[4]/text()').extract( )[0] baseItem['product_type'] = sel.xpath( '//div[contains(@class, "bread bread_bar")]//a[3]/text()').extract( )[0] yield baseItem
def handle_parse_item(self, response, item): sel = Selector(response) item['show_product_id'] = str( re.search('productID: \"(\d+)\"', response.body).group(1)).strip() item['brand'] = str( re.search('productBrand: \"(.+)\"', response.body).group(1)).strip() item['title'] = sel.xpath( './/h1[@data-track="product-title"]/text()').extract()[0].strip() item['desc'] = ''.join( sel.xpath('//div[@itemprop="description"]/p').extract()).strip() item['current_price'] = sel.xpath( '//span[@class="price"]/text()').extract()[0].strip() list_price_search = re.search('rrp: .+\&\#36\;([\d\.]+).+', response.body) if list_price_search: item['list_price'] = list_price_search.group(1) else: item['list_price'] = item['current_price'] images = [] image_divs = sel.xpath( '//div[@class="product-thumb-box productImageZoom__thumbnailContainer "]' ) if not image_divs: return for image_div in image_divs: imageItem = ImageItem() imageItem['thumbnail'] = image_div.xpath('./img/@src').extract()[0] imageItem['image'] = image_div.xpath( './parent::*/@href').extract()[0] images.append(imageItem) color_names = sel.xpath( '//select[@id="opts-2"]/option[position()>1]/text()').extract() if len(color_names) > 1: return if not color_names: color_names = ['One Color'] item['colors'] = color_names color = Color() color['type'] = 'color' color['from_site'] = item['from_site'] color['show_product_id'] = item['show_product_id'] color['images'] = images color['name'] = color_names[0] color['cover'] = images[0]['thumbnail'] yield color skus = [] sizes = sel.xpath( '//select[@id="opts-1"]/option[position()>1]/text()').extract() if not sizes: sizes = ['One Size'] item['sizes'] = sizes for size in sizes: for color_name in color_names: skuItem = SkuItem() skuItem['type'] = "sku" skuItem['from_site'] = item['from_site'] skuItem['id'] = item[ 'show_product_id'] + '-' + color_name + '-' + size skuItem['show_product_id'] = item['show_product_id'] skuItem['current_price'] = item['current_price'] skuItem['list_price'] = item['list_price'] skuItem['size'] = size skuItem['color'] = color_name skus.append(skuItem) item['skus'] = skus item['dimensions'] = ['size'] yield item
def handle_parse_item(self, response, item): sel = Selector(response) if len(sel.xpath('//form[@id="product-form"]//meta').extract()) > 1: return if len(sel.xpath('//div[@class="sold-out-details"]')) > 0: return item['show_product_id'] = sel.xpath( '//div[@class="product-code"]/span/text()').extract()[0].strip() imgs = sel.xpath( '//div[@class="container-imagery"]//ul[@class="thumbnails no-carousel"]/li/img/@src' ).extract() if len(imgs) == 0: imgs = sel.xpath( '//div[@class="container-imagery"]//ul[@class="swiper-wrapper"]/li/img/@src' ).extract() images = [] for img in imgs: if 'http:' not in img: img = 'http:' + img if 'xs.jpg' in img: img = img.replace('xs.jpg', 'pp.jpg') imageItem = ImageItem() imageItem['image'] = img imageItem['thumbnail'] = img.replace('pp.jpg', 'm.jpg') images.append(imageItem) colorItem = Color() colorItem['images'] = images colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = item['show_product_id'] colorItem['name'] = 'One Color' colorItem['cover'] = images[0]['image'].replace('pp.jpg', 'xs.jpg') # colorItem['cover'] = images[0]['image'].split('_')[0] + '_sw.jpg' # print colorItem['cover'] # req = requests.get(colorItem['cover']) # if not req.ok: # colorItem['cover'] = images[0]['image'].replace('pp.jpg', 'xs.jpg') yield colorItem price = int( sel.xpath('//form[@id="product-form"]/meta/@data-price-full'). extract()[0]) / 100 if len(sel.xpath('//select-dropdown[@class="sku"]/@options')) > 0: sku_str = sel.xpath( '//select-dropdown[@class="sku"]/@options').extract()[0] skus = json.loads(sku_str) item['skus'] = [] sizes = [] for sku in skus: skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['list_price'] = price skuItem['current_price'] = price skuItem['color'] = 'One Color' skuItem['size'] = sku['data']['size'] sizes.append(sku['data']['size']) skuItem['id'] = sku['id'] skuItem['from_site'] = item['from_site'] if sku['stockLevel'] == 'In_Stock' or sku[ 'stockLevel'] == 'Low_Stock': skuItem['is_outof_stock'] = False else: skuItem['is_outof_stock'] = True item['skus'].append(skuItem) else: item['skus'] = [] skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['list_price'] = price skuItem['current_price'] = price skuItem['color'] = 'One Color' sizes = ['One Size'] skuItem['size'] = 'One Size' skuItem['id'] = sel.xpath( '//input [@class="sku"]/@value').extract()[0] stock_level = sel.xpath( '//input [@class="sku"]/@data-stock').extract()[0] if stock_level == 'In_Stock' or stock_level == 'Low_Stock': skuItem['is_outof_stock'] = False else: skuItem['is_outof_stock'] = True skuItem['from_site'] = item['from_site'] item['skus'].append(skuItem) item['gender'] = self.gender item['colors'] = ['One Color'] item['sizes'] = sizes item['desc'] = '' if len(sel.xpath('//widget-show-hide[@id="accordion-1"]//ul/li')) > 0: item['desc'] = item['desc'] + sel.xpath( '//widget-show-hide[@id="accordion-1"]//ul/li').extract()[0] if len(sel.xpath('//widget-show-hide[@id="accordion-2"]//ul/li')) > 0: item['desc'] = item['desc'] + sel.xpath( '//widget-show-hide[@id="accordion-2"]//ul/li').extract()[0] if len(sel.xpath('//widget-show-hide[@id="accordion-2"]//p')) > 0: item['desc'] = item['desc'] + sel.xpath( '//widget-show-hide[@id="accordion-2"]//p').extract()[0] product_items = sel.xpath( '//widget-show-hide[@name="Editor\'s Notes"]/div[@class="show-hide-content"]/div/p/a' ) if len(product_items) > 0: related_items_id = [] for product_item in product_items: product_id = product_item.xpath('./@href').extract()[0].split( '/')[-1] related_items_id.append(product_id) if related_items_id: item['related_items_id'] = related_items_id media_url = 'https://video.net-a-porter.com/videos/productPage/' + item[ 'show_product_id'] + '_detail.mp4' try: req = requests.head(media_url) if req.ok: item['media_url'] = media_url except Exception as e: logging.error('error media url: ' + media_url + ' error msg: ' + str(e)) yield item
def handle_parse_item(self, response, item): sel = Selector(response) detail_str = re.search('<script>ReactDOM\.render\(React\.createElement\(ProductDesktop\,[\s]*(.+)[\s]*\)\,[\s]*document\.getElementById', response.body) if detail_str is None: return context = execjs.compile(''' var json = %s function getJson(){ return json; } ''' % detail_str.group(1)) detail_json = context.call('getJson') goods_detail = detail_json['initialData']['Model']['StyleModel'] if goods_detail['IsAvailable'] == False: return item['desc'] = goods_detail['Description'] item['show_product_id'] = goods_detail['Id'] item['dimensions'] = ['size', 'color'] # if len(sel.xpath('//div[@class="size-chart"]/a/@href'))>0: # item['size_info'] = sel.xpath('//div[@class="size-chart"]/a/@href').extract()[0] # else: item['size_info'] = '' color_details = goods_detail['StyleMedia'] color_names = [] color_images = {} image_keys = ['Zoom', 'MaxLarge', 'Large', 'Medium'] for color_detail in color_details: if color_detail['ColorName'] == '': continue if color_detail['ColorName'] not in color_images.keys(): color_images[color_detail['ColorName']]=[] imageItem = ImageItem() # imageItem['image'] = color_detail['ImageMediaUri']['Gigantic'] # if color_detail['ImageMediaUri']['Gigantic'] == "": for image_key in image_keys: imageItem['image'] = color_detail['ImageMediaUri'][image_key] if not color_detail['ImageMediaUri'][image_key] or 'g.nordstromimage.com' in color_detail['ImageMediaUri'][image_key]: continue else: imageItem['image'] = color_detail['ImageMediaUri'][image_key] break imageItem['thumbnail'] = color_detail['ImageMediaUri']['Large'] if imageItem['thumbnail'] or 'g.nordstromimage.com' in imageItem['thumbnail']: imageItem['thumbnail'] = color_detail['ImageMediaUri']['Medium'] color_images[color_detail['ColorName']].append(imageItem) if len(color_images) > 0: for color_name, images in color_images.items(): color = Color() color['type'] = 'color' color['from_site'] = self.name color['show_product_id'] = item['show_product_id'] color['images'] = images color['name'] = color_name color['cover'] = images[0]['thumbnail'] yield color else: for color_detail in color_details: imageItem = ImageItem() for image_key in image_keys: imageItem['image'] = color_detail['ImageMediaUri'][image_key] if color_detail['ImageMediaUri'][image_key] == "" or 'g.nordstromimage.com' in color_detail['ImageMediaUri'][image_key]: continue else: imageItem['image'] = color_detail['ImageMediaUri'][image_key] break imageItem['thumbnail'] = color_detail['ImageMediaUri']['Large'] if imageItem['thumbnail'] or 'g.nordstromimage.com' in imageItem['thumbnail']: imageItem['thumbnail'] = color_detail['ImageMediaUri']['Medium'] images = [imageItem] color_as = sel.xpath('//ul[@class="swatch-list"]/li/a/span/img') color_names = [] for color_a in color_as: color = Color() color['type'] = 'color' color['from_site'] = self.name color['show_product_id'] = item['show_product_id'] color_name = color_a.xpath('./@alt').extract()[0] if 'swatch image' in color_name: color_name = color_name.replace('swatch image','') color['name'] = color_name.strip() color_names.append(color_name) color['images'] = images color['cover'] = color_a.xpath('./@src').extract()[0] yield color list_price_list = [] sizes = [] skus = [] widths = [] for item_group in goods_detail['ChoiceGroups']: list_price_temp = item_group['Price']['OriginalPrice'] if '–' in list_price_temp: list_price_temp = list_price_temp.split('–')[1].strip() list_price_list.append(list_price_temp) if 'Size' not in item_group.keys() or item_group['Size'] is None: sizes = ['One Size'] continue if 'Width' in item_group.keys() and item_group['Width'] is not None: if 'width' not in item['dimensions']: item['dimensions'].append('width') for width_group in item_group['Width']: if width_group['Value'] not in widths: widths.append(width_group['Value']) for size_group in item_group['Size']: if size_group['Value'] not in sizes: sizes.append(size_group['Value']) if widths: size_width = {'size': sizes, 'width': widths} else: size_width = {'size': sizes} sku_list_price = max(list_price_list) for sku_detail in goods_detail['Skus']: skuItem = {} skuItem['type'] = "sku" skuItem['from_site'] = self.name skuItem['id'] = sku_detail['Id'] skuItem['show_product_id'] = item['show_product_id'] skuItem['list_price'] = sku_list_price skuItem['current_price'] = sku_detail['Price'] skuItem['color'] = sku_detail['Color'] if sku_detail['Size']: skuItem['size'] = {'size': sku_detail['Size']} if 'Width' in sku_detail.keys() and sku_detail['Width'] is not None: skuItem['size']['width'] = sku_detail['Width'] else: skuItem['size'] = {'size': 'One Size'} skus.append(skuItem) if len(color_images) > 0: item['colors'] = color_images.keys() else: item['colors'] = color_names item['sizes'] = size_width item['skus'] = skus # if len(sel.xpath('//div[@class="size-chart"]/a/@href'))>0: # size_chart_url = sel.xpath('//div[@class="size-chart"]/a/@href').extract()[0] # yield Request(size_chart_url, callback=self.parse_size_chart, meta={'item': item}) # else: yield item
def handle_parse_item(self, response, item): sel = Selector(response) item['show_product_id'] = re.search('(\d+)\.html', response.url).group(1) item['title'] = sel.xpath( '//div[@class="product-name"]/h1/text()').extract()[0].strip() brand = re.search('brand: \'(.+)\'', response.body) if not brand: item['brand'] = 'pharmacyonline' else: item['brand'] = brand.group(1) img = re.search('imgUrl: \'(.+)\'', response.body).group(1) images = [] imageItem = ImageItem() imageItem[ 'thumbnail'] = img + '?imageMogr2/thumbnail/380x380/extent/380x380/background/d2hpdGU=' imageItem['image'] = img images.append(imageItem) # item['cover'] = images[0]['thumbnail'] item['colors'] = ['One Color'] color = Color() color['type'] = 'color' color['from_site'] = item['from_site'] color['show_product_id'] = item['show_product_id'] color['images'] = images color['name'] = 'One Color' color['cover'] = images[0][ 'image'] + '?imageMogr2/thumbnail/100x100/extent/100x100/background/d2hpdGU=' yield color item['desc'] = sel.xpath( '//div[@class="product-collateral"]').extract()[0] current_price = sel.xpath( '//div[@class="DetailNoDis PriceNow last_price_sing"]/span/text()') if len(current_price) > 0: item['current_price'] = current_price.extract()[0] item['list_price'] = item['current_price'] else: item['current_price'] = sel.xpath( '//div[@class="DetailPriceContain clearfix"]//div[@class="PriceNow"]/text()' ).extract()[0].strip() item['list_price'] = sel.xpath( '//div[@class="DetailPriceContain clearfix"]//p[@class="PriceWas"]/text()' ).extract()[0].strip() skus = [] item['sizes'] = ['One Size'] skuItem = SkuItem() skuItem['type'] = "sku" skuItem['from_site'] = item['from_site'] sku_id = sel.xpath( '//div[@class="DetailSku"]/text()').extract()[0].strip() skuItem['id'] = re.search('(\d+)', sku_id).group(1) skuItem['show_product_id'] = item['show_product_id'] skuItem['current_price'] = item['current_price'] skuItem['list_price'] = item['list_price'] skuItem['size'] = 'One Size' skuItem['color'] = 'One Color' skus.append(skuItem) item['skus'] = skus item['dimensions'] = ['size'] if len(item['show_product_id']) > 6: product_id = item['show_product_id'][1:] else: product_id = item['show_product_id'] stock_url = 'http://cn.pharmacyonline.com.au/pt_catalog/index/checkQty?product_id=' + product_id yield Request(stock_url, callback=self.parse_stock, meta={"item": item}, dont_filter=True)
def handle_parse_item(self, response, item): sel = Selector(response) product_str = re.search('var dataLayer\s*=\s*(.+?);', response.body).group(1) product_json = json.loads(product_str) product = product_json['products'][0] item['brand'] = 'Tiffany' item['title'] = product['name'] if not product['price']: return item['list_price'] = product['price'] item['current_price'] = product['price'] item['desc'] = sel.xpath( '//div[@id="drawerDescription"]/div/div').extract()[0] item['cover'] = sel.xpath('//meta[@property="og:image"]/@content' ).extract()[0] + self.cover_img_surffix if product['stockStatus'] == 'out of stock': return skus = [] sizes = [] if not sel.xpath( '//select[@id="ctlSkuGroupType1_selItemList"]/option'): item['show_product_id'] = product['sku'] skuItem = SkuItem() skuItem['type'] = "sku" skuItem['from_site'] = item['from_site'] skuItem['size'] = 'One Size' sizes = ['One Size'] skuItem['color'] = 'One Color' skuItem['id'] = item['show_product_id'] + '-' + skuItem[ 'color'] + '-' + skuItem['size'] skuItem['show_product_id'] = item['show_product_id'] skuItem['current_price'] = item['current_price'] skuItem['list_price'] = item['list_price'] skus.append(skuItem) else: item['show_product_id'] = product['groupSku'] size_options = sel.xpath( '//select[@id="ctlSkuGroupType1_selItemList"]/option') for size_option in size_options: skuItem = SkuItem() skuItem['type'] = "sku" skuItem['from_site'] = item['from_site'] if not size_option.xpath('./text()').extract(): skuItem['size'] = 'One Size' else: skuItem['size'] = size_option.xpath( './text()').extract()[0] sizes.append(skuItem['size']) skuItem['color'] = 'One Color' skuItem['id'] = item['show_product_id'] + '-' + skuItem[ 'color'] + '-' + skuItem['size'] skuItem['show_product_id'] = item['show_product_id'] skuItem['current_price'] = item['current_price'] skuItem['list_price'] = item['list_price'] skus.append(skuItem) images = [] imageItem = ImageItem() imageItem['thumbnail'] = sel.xpath( '//meta[@property="og:image"]/@content').extract( )[0] + self.large_img_surffix imageItem['image'] = sel.xpath('//meta[@property="og:image"]/@content' ).extract()[0] + self.large_img_surffix images.append(imageItem) color = Color() color['type'] = 'color' color['from_site'] = item['from_site'] color['show_product_id'] = item['show_product_id'] color['images'] = images color['name'] = 'One Color' color['cover'] = item['cover'] yield color item['colors'] = ['One Color'] item['sizes'] = sizes item['skus'] = skus item['dimensions'] = ['size'] yield item
def handle_parse_item(self, response, item): match = re.search( r'<script type\=\"application\/json\">({"ProductDetails".+?)<\/script>', response.body) print match.group(1) sel = Selector(response) if match is None: return context = execjs.compile(''' var json = %s function getJson(){ return json; } ''' % match.group(1)) product_json = context.call('getJson') main_product = product_json['ProductDetails']['main_products'][0] item['brand'] = main_product['brand_name']['label'] item['title'] = main_product['short_description'] show_product_id = main_product['product_code'] item['show_product_id'] = show_product_id item['desc'] = main_product['description'] list_price = main_product['price']['list_price']['usd_currency_value'] if re.findall('\-', list_price): re.search('([\d\.]+)\s*\-', list_price).group(1) else: item['list_price'] = list_price sale_price = main_product['price']['sale_price']['usd_currency_value'] if re.findall('\-', sale_price): re.search('([\d\.]+)\s*\-', sale_price).group(1) else: item['current_price'] = sale_price item['dimensions'] = ['size'] skus = [] sizes = {} sizes['size'] = [] color_names = [] colors = main_product['colors']['colors'] handle_color_map = {} if len(colors) > 0: for color in colors: handle_color_map[color['id']] = color['label'] handle_size_map = {} if len(main_product['sizes']['sizes']) == 0: sizes['size'].append('onesize') else: for size in main_product['sizes']['sizes']: handle_size_map[size['id']] = size['value'] sizes['size'].append(size['value']) image_prefix = 'http:' + main_product['media'][ 'images_server_url'] + main_product['media']['images_path'] if len(colors) == 0: color_name = 'onecolor' color_names.append(color_name) common_images = main_product['media']['images'] images = [] for common_image in common_images: imageItem = ImageItem() imageItem[ 'image'] = image_prefix + common_image + '?wid=970&hei=1293&fmt=jpg' imageItem[ 'thumbnail'] = image_prefix + common_image + '?wid=396&hei=528&fmt=jpg' images.append(imageItem) first_thumbnail = images[0]['thumbnail'] colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = item['show_product_id'] colorItem['images'] = images colorItem['name'] = color_name colorItem['cover'] = first_thumbnail colorItem['version'] = '1' yield colorItem else: common_images = main_product['media']['images'] for color in colors: color_name = color['label'] color_names.append(color_name) images = [] imageItem = ImageItem() imageItem['image'] = image_prefix + color[ 'colorize_image_url'] + '?wid=970&hei=1293&fmt=jpg' imageItem['thumbnail'] = image_prefix + color[ 'colorize_image_url'] + '?wid=396&hei=528&fmt=jpg' images.append(imageItem) first_thumbnail = images[0]['thumbnail'] for common_image in common_images: imageItem = ImageItem() imageItem[ 'image'] = image_prefix + common_image + '?wid=970&hei=1293&fmt=jpg' imageItem[ 'thumbnail'] = image_prefix + common_image + '?wid=396&hei=528&fmt=jpg' images.append(imageItem) colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = item['show_product_id'] colorItem['images'] = images colorItem['name'] = color_name colorItem['version'] = '1' if len(color['value']) > 0: if re.findall('\#', color['value']): colorItem['cover_style'] = color['value'] else: cover_img_str = sel.xpath( '//li[@class="product-color-options__value" and @data-colorid=' + str(color["id"]) + ']/@style').extract() cover_unavi_str = sel.xpath( '//li[@class="product-color-options__value product-color-options__value--unavailable" and @data-colorid=' + str(color["id"]) + ']/@style').extract() cover_sel_str = sel.xpath( '//li[@class="product-color-options__value product-color-options__value--selected" and @data-colorid=' + str(color["id"]) + ']/@style').extract() cover_hid_str = sel.xpath( '//li[@class="product-color-options__value is-hidden" and @data-colorid=' + str(color["id"]) + ']/@style').extract() if len(cover_img_str) > 0: cover_img = re.search('\((.+)\)', cover_img_str[0]).group(1) colorItem['cover'] = 'http:' + cover_img elif len(cover_unavi_str) > 0: cover_img_str = cover_unavi_str[0] cover_img = re.search('\((.+)\)', cover_img_str).group(1) colorItem['cover'] = 'http:' + cover_img elif len(cover_sel_str) > 0: cover_img_str = cover_sel_str[0] cover_img = re.search('\((.+)\)', cover_img_str).group(1) colorItem['cover'] = 'http:' + cover_img elif len(cover_hid_str) > 0: cover_img_str = cover_hid_str[0] cover_img = re.search('\((.+)\)', cover_img_str).group(1) colorItem['cover'] = 'http:' + cover_img else: colorItem['cover'] = first_thumbnail else: colorItem['cover'] = first_thumbnail yield colorItem item['colors'] = color_names for sku in main_product['skus']['skus']: sku_id = sku['sku_id'] if sku_id == 'DUMMY': continue if sku['color_id'] == -1: color_name = 'onecolor' else: color_name = handle_color_map[sku['color_id']] if sku['size_id'] == -1: size = 'onesize' else: size = handle_size_map[sku['size_id']] skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['from_site'] = item['from_site'] skuItem['id'] = sku_id skuItem['size'] = size skuItem['color'] = color_name if sku['status_alias'] == 'soldout' or sku[ 'status_alias'] == 'waitlist': skuItem['is_outof_stock'] = True else: skuItem['is_outof_stock'] = False if len(sku['price']['sale_price']['usd_currency_value']) > 0: skuItem['current_price'] = sku['price']['sale_price'][ 'usd_currency_value'] else: continue if len(sku['price']['list_price']['usd_currency_value']) > 0: skuItem['list_price'] = sku['price']['list_price'][ 'usd_currency_value'] else: continue skus.append(skuItem) item['sizes'] = sizes item['skus'] = skus if main_product['size_guide_link']['enabled'] == True: sizeInfo = main_product['size_guide_link']['url'] findQ = sizeInfo.find("?") if findQ != -1: item['size_info'] = sizeInfo[:findQ] else: item['size_info'] = sizeInfo yield item
def handle_parse_item(self, response, item): sel = Selector(response) item['current_price'] = product_price[6:] item['list_price'] = rrp_prices[1][6:] if 'show_product_id' not in item.keys(): item['show_product_id'] = eval(re.search(r'productID:(.*),', response.body).group(1)) if 'brand' not in item.keys(): item['brand'] = eval(re.search(r'productBrand:(.*),', response.body).group(1)) if item['brand'] == '': item['brand'] = 'allsole' desc = sel.xpath(".//div[@itemprop='description']/*").extract() if len(desc) > 0: item['desc'] = desc[0] else: item['desc'] = '' sizeDoms = sel.xpath(".//select[@id='opts-1']/option[string-length(@value) > 0]") item['dimensions'] = ['size'] item['skus'] = [] if len(sizeDoms.extract()): colorDoms = sel.xpath(".//select[@id='opts-2']/option[@value>0]") if len(colorDoms.extract()): color = colorDoms.xpath("./text()").extract()[0] item["colors"] = [color] else: item["colors"] = ["onecolor"] item['sizes'] = [] for dom in sizeDoms: curr_size = dom.xpath("./text()").extract()[0] item['sizes'].append(curr_size) skuItem = {} skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['id'] = item["colors"][0] + '*' + curr_size skuItem['color'] = item["colors"][0] skuItem['size'] = curr_size ##skuItem['size_num'] = re.search(r'\d+', curr_size).group(0) skuItem['from_site'] = self.name skuItem['list_price'] = item['list_price'] skuItem['current_price'] = item['current_price'] skuItem['is_outof_stock'] = False item['skus'].append(skuItem) else: item['sizes'] = ['onesize'] item['colors'] = ['onecolor'] skuItem = {} skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['id'] = item['colors'][0] + '*' + item['sizes'][0] skuItem['color'] = item['colors'][0] skuItem['size'] = item['sizes'][0] skuItem['from_site'] = self.name skuItem['list_price'] = item['list_price'] skuItem['current_price'] = item['current_price'] skuItem['is_outof_stock'] = False item['skus'].append(skuItem) imgDoms = sel.xpath(".//div[@class='media']//li[@class='n-unit']") images = [] if len(imgDoms.extract()): for dom in imgDoms: imageItem = ImageItem() imageItem['image'] = dom.xpath("./a/@href").extract()[0] imageItem['thumbnail'] = dom.xpath("./a/div/img/@src").extract()[0] images.append(imageItem) colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = self.name colorItem['show_product_id'] = item['show_product_id'] colorItem['name'] = item['colors'][0] colorItem['cover'] = images[0]['thumbnail'] colorItem['images'] = images yield colorItem yield item
def handle_parse_item(self, response, item): if 'Error' in response.url: return sel = Selector(response) item['category'] = sel.xpath( '//section[@id="Breadcrumbs"]/nav/a[last()]/text()').extract()[0] show_product_id = sel.xpath('//span[@id="Sku"]/text()').extract()[-1] details_str = ''.join( re.search('sf\.productDetail\.init\((\{[\s\S]+?\})\)\;', response.body).group(1)) context = execjs.compile(''' var skus = %s; function getSkus(){ return skus; } ''' % details_str) item['skus'] = [] details = context.call('getSkus') skus_json = details['variants'] images_json = details['images'] images = [] sizes = [] width = [] dimensions_dict = {} is_in_stock = False for children in details['children']: if children['c'] == show_product_id: color_name = children['v'] is_in_stock = True if not is_in_stock: return list_price = ''.join( sel.xpath( '//div[@id="Price"]/div[@class="floatLeft productPriceInfo"]/span[@class="origPrice"]//span/text()' ).extract()).strip() current_price = ''.join( sel.xpath( '//div[@id="Price"]/span[@class="floatLeft productPrice "]/text() | //div[@id="Price"]/span[@class="floatLeft productPrice "]/span/text()' ).extract()).strip() if not list_price: list_price = ''.join( sel.xpath( '//div[@id="Price"]//span/text()').extract()).strip() current_price = list_price for sku_json in skus_json: skuItem = {} skuItem['type'] = 'sku' skuItem['id'] = sku_json['v'] skuItem['show_product_id'] = show_product_id skuItem['list_price'] = list_price skuItem['current_price'] = current_price skuItem['from_site'] = item['from_site'] skuItem['is_outof_stock'] = False skuItem['size'] = {} for sku in sku_json['a']: if sku['n'] == 'custom.color': skuItem['color'] = sku['v'] else: if sku['n'] not in dimensions_dict.keys(): dimensions_dict[sku['n']] = [] if sku['v'] not in dimensions_dict[sku['n']]: dimensions_dict[sku['n']].append(sku['v']) skuItem['size'][sku['n']] = sku['v'] # if sku['n'] == 'size': # skuItem['size']['size'] = sku['v'] # dimensions_dict['size'].append(sku['v']) # if sku['n'] == 'width': # skuItem['size']['width'] = sku['v'] # dimensions_dict['width'].append(sku['v']) if skuItem['size']['size'] not in sizes: sizes.append(skuItem['size']['size']) if 'width' in skuItem['size'].keys( ) and skuItem['size']['width'] not in width: width.append(skuItem['size']['width']) item['skus'].append(skuItem) for img_json in images_json: if img_json['c'] == show_product_id: imageItem = ImageItem() imageItem['thumbnail'] = img_json['f'] imageItem['image'] = img_json[ 'f'][:img_json['f'].find('?')] + '?wid=1280&hei=900' images.append(imageItem) if not images: return colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = show_product_id colorItem['images'] = images colorItem['cover'] = images[0]['thumbnail'] colorItem['name'] = color_name yield colorItem item['show_product_id'] = show_product_id item['dimensions'] = dimensions_dict.keys() if len(sel.xpath('//div[@id="Description"]/p/text()')) > 0: item['desc'] = sel.xpath( '//div[@id="Description"]/p/text()').extract()[0].strip() else: item['desc'] = ''.join( sel.xpath('//ul[@class="productFeaturesDetails"]//li').extract( )).strip() item['colors'] = [color_name] item['sizes'] = dimensions_dict yield item
def parse_color(self, response): item = response.meta['item'] color_forms = response.meta['color_forms'] # print color_forms colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = item['show_product_id'] response_body = re.sub(r'null', 'None', response.body) color_form_response = eval(response_body) if 'images' in color_form_response.keys(): thumbnail = color_form_response['images'][0]['name'] if len(color_form_response['images']) < 4: return image = color_form_response['images'][3]['name'] if '/600' not in image: image = re.sub('/[\d]+/[\d]+', '/600/600', image) color_name = color_form_response['variations'][0]['options'][0][ 'name'] cover_style = color_form_response['variations'][0]['options'][0][ 'value'] price_str = color_form_response['price'] m = re.search(r'\d+\.\d+', price_str) if m: current_price = m.group(0) imageItem = ImageItem() imageItem['thumbnail'] = 'https://s4.thcdn.com/' + thumbnail imageItem['image'] = 'https://s4.thcdn.com/' + image images = [] images.append(imageItem) colorItem['name'] = color_name colorItem['images'] = images colorItem['cover_style'] = cover_style yield colorItem if 'skus' not in item.keys(): item['skus'] = [] skuItem = {} skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['id'] = color_name + '*' + 'onesize' skuItem['list_price'] = item['list_price'] skuItem['current_price'] = current_price skuItem['from_site'] = item['from_site'] skuItem['color'] = color_name skuItem['size'] = 'onesize' skuItem['is_outof_stock'] = False item['skus'].append(skuItem) if len(color_forms) == 0: yield item else: # import pdb;pdb.set_trace() color_ajax_url = 'https://www.lookfantastic.com/variations.json?productId=' + str( item['show_product_id']) color_form_value = color_forms.pop(0) # for color_form_value in color_form: form_data = { 'option1': str(color_form_value), 'selected': '1', 'variation1': '4' } # import pdb;pdb.set_trace() yield FormRequest(color_ajax_url, formdata=form_data, callback=self.parse_color, meta={ 'item': item, 'colorItem': colorItem, 'color_forms': color_forms })
def handle_parse_item(self, response, item): sel = Selector(response) soldout_span = sel.xpath( '//div[@class="cta-container"]//span[contains(@class, "soldout")]/text()' ).extract() if len(soldout_span) > 0: return # siteObjStr = "".join(re.findall(r'(var siteObj = [\s\S]*exitting = false;)', response.body)) # context = execjs.compile(''' # %s # function getSiteObj(){ # return siteObj; # } # ''' % siteObjStr) # tmpStr = json.dumps(context.call('getSiteObj')) # siteObj = json.loads(tmpStr) # product_id = siteObj['productID'] # product_title = siteObj['productTitle'] # product_price = siteObj['productPrice'] # product_brand = siteObj['productBrand'] # product_rrp = siteObj['rrp'] product_id = eval( re.search(r'productID:(.*),', response.body).group(1)) # product_title = eval(re.search(r'productTitle:(.*),', response.body).group(1)) product_price = eval( re.search(r'productPrice:(.*),', response.body).group(1)) product_brand = eval( re.search(r'productBrand:(.*),', response.body).group(1)) product_rrp = eval(re.search(r'rrp:(.*),', response.body).group(1)) if len(sel.xpath(".//h1[@itemprop='name']/text()")) > 0: product_title = sel.xpath( ".//h1[@itemprop='name']/text()").extract()[0] else: current_url = response.url product_title = ' '.join(current_url.split('/')[3].split('-')) rrp_dr = re.compile(r'<[^>]+>', re.S) rrp = rrp_dr.sub('', product_rrp) rrp_prices = re.split(r': ', rrp) if len( sel.xpath( ".//*[@id='health-beauty']//div[@itemprop='description']"). extract()) > 0: product_desc = sel.xpath( ".//*[@id='health-beauty']//div[@itemprop='description']" ).extract()[0] else: product_desc = '' if product_id: item['show_product_id'] = product_id item['title'] = product_title item['current_price'] = product_price.replace('£', '') item['list_price'] = rrp_prices[1].replace('£', '') item['desc'] = product_desc item['brand'] = product_brand item['dimensions'] = ['size'] item['sizes'] = ['onesize'] item['colors'] = ['onecolor'] if re.findall(r'men', item['brand']): item['gender'] = 'men' else: return thumbnails = sel.xpath( ".//ul[@class='product-thumbnails nav-items product-large-view-thumbs productImageZoom__thumbs']/li//div[contains(@class, 'product-thumb-box')]/img/@src" ).extract() #thumbnails = sel.xpath(".//div[@class='product-thumb-box']/img/@src").extract() # viewimages = sel.xpath(".//div[@class='product-thumb-box']/parent::*/@href").extract() #480*480 viewimages = sel.xpath( ".//ul[@class='product-thumbnails nav-items product-large-view-thumbs productImageZoom__thumbs']/li/a/@href" ).extract() #600*600 images = [] color_cover = None if len(thumbnails) > 0: imgKey = 0 for imgVal in thumbnails: imageItem = ImageItem() imageItem['image'] = handle_no_http(viewimages[imgKey].strip()) imageItem['thumbnail'] = handle_no_http(imgVal) if not color_cover: color_cover = imageItem['thumbnail'] images.append(imageItem) imgKey += 1 color_form = sel.xpath( '//div[@class="variation-dropdowns"]/form//option/@value').extract( )[1:] color_names = sel.xpath( '//div[@class="variation-dropdowns"]/form//option/text()').extract( )[1:] size_or_color_str = sel.xpath( '//div[@class="variation-dropdowns"]/form//label/text()').extract( ) if len(size_or_color_str) > 0: size_or_color = size_or_color_str[0] if len(color_form) > 0 and size_or_color == 'Shade:': item['colors'] = color_names color_ajax_url = 'https://www.lookfantastic.com/variations.json?productId=' + str( item['show_product_id']) color_form_value = color_form.pop(0) # for color_form_value in color_form: form_data = { 'option1': str(color_form_value), 'selected': '1', 'variation1': '4' } # import pdb;pdb.set_trace() yield FormRequest(color_ajax_url, formdata=form_data, callback=self.parse_color, meta={ 'item': item, 'color_forms': color_form }) elif len(color_form) > 0 and size_or_color == 'Size:': item['sizes'] = color_names #pop color colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = item['show_product_id'] colorItem['name'] = 'onecolor' colorItem['images'] = images colorItem['cover'] = color_cover yield colorItem size_ajax_url = 'https://www.lookfantastic.com/variations.json?productId=' + str( item['show_product_id']) color_form_value = color_form.pop(0) # for color_form_value in color_form: form_data = { 'option1': str(color_form_value), 'selected': '1', 'variation1': '1' } # import pdb;pdb.set_trace() yield FormRequest(size_ajax_url, formdata=form_data, callback=self.parse_size, meta={ 'item': item, 'color_forms': color_form }) else: colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = product_id colorItem['images'] = images colorItem['cover'] = color_cover colorItem['name'] = 'onecolor' yield colorItem item['skus'] = [] skuItem = {} skuItem['type'] = 'sku' skuItem['show_product_id'] = product_id skuItem['id'] = product_id skuItem['list_price'] = item['list_price'] skuItem['current_price'] = item['current_price'] skuItem['from_site'] = item['from_site'] skuItem['color'] = 'onecolor' skuItem['size'] = 'onesize' skuItem['is_outof_stock'] = False item['skus'].append(skuItem) # yield skuItem yield item
def handle_parse_item(self, response, item): sel = Selector(response) if len(sel.xpath('//div[@id="sold-out-div"]')) > 0: return if len(sel.xpath('//div[@class="price_box sale"]'))>0: list_pirce = sel.xpath('//div[@class="price_box sale"]/span[@class="price"]/text()').extract()[0].strip() current_price = sel.xpath('//div[@class="price_box sale"]/span[@class="discount_price"]/text()').extract()[0].strip() print 1111111111 elif len(sel.xpath('//div[@class="price_box"]/span[@class="discount_price"]/text()')) > 0: list_pirce = sel.xpath('//div[@class="price_box"]/span[@class="discount_price"]/text()').extract()[0].strip() current_price = list_pirce print 2222222222222 elif len(sel.xpath('//div[@id="tr-pdp-price--sale"]')) > 0: list_pirce = sel.xpath('//div[@id="tr-pdp-price--sale"]/span[@class="prices__retail-strikethrough u-margin-l--lg"]/text()').extract()[0] current_price = sel.xpath('//div[@id="tr-pdp-price--sale"]/span[@class="prices__markdown"]/text()').extract()[0] print 33333333333333 elif len(sel.xpath('//div[@class="product_info"]/div[@class="eagle"]/div[@class="prices"]/span/text()'))>0: list_pirce = sel.xpath('//div[@class="product_info"]/div[@class="eagle"]/div[@class="prices"]/span/text()').extract()[0].strip() print list_pirce current_price = list_pirce print 444444444444444444 else: return if 'USD' in list_pirce: list_pirce = list_pirce.replace('USD', '') if 'USD' in current_price: current_price = current_price.replace('USD', '') item['show_product_id'] = sel.xpath('//span[@id="find-your-size"]/a/@data-code').extract()[0] item['desc'] = sel.xpath('//div[@id="details"]/ul').extract()[0] item['dimensions'] = ['size', 'color'] images = [] image_divs = sel.xpath('.//div[@class="product-detail-image-zoom"]') for image_div in image_divs: imageItem = ImageItem() imageItem['image'] = image_div.xpath('./img/@src').extract()[0] imageItem['thumbnail'] = re.sub('fw/z', 'fw/p', imageItem['image']) images.append(imageItem) if not images: return color = Color() color['type'] = 'color' color['from_site'] = self.name color['show_product_id'] = item['show_product_id'] color['images'] = images if len(sel.xpath('//select[@id="color-select"]'))>0: color_name = sel.xpath('//select[@id="color-select"]/option[1]/@value').extract()[0].strip() color['name'] = color_name else: color_name_ext = sel.xpath('//div[@class="color_dd"]/div[@class="title one_sizeonly"]/text()').extract() if color_name_ext: color_name = color_name_ext[0].strip() else: color_name = 'One Color' color['name'] = color_name cover = re.sub('fw/z', 'fw/c', images[0]['thumbnail']) color['cover'] = cover yield color skus = [] sizes = [] if len(sel.xpath('//div[@class="size_dd"]/div[@class="title one_sizeonly"]'))>0: skuItem = {} skuItem['type'] = "sku" skuItem['from_site'] = self.name size = sel.xpath('//div[@class="size_dd"]/div[@class="title one_sizeonly"]/text()').extract()[0].strip() skuItem['size'] = size sizes.append(size) skuItem['id'] = color_name + size skuItem['show_product_id'] = item['show_product_id'] skuItem['list_price'] = list_pirce skuItem['current_price'] = current_price skuItem['color'] = color_name skus.append(skuItem) else: for size_opt in sel.xpath('//select[@id="size-select"]/option[position()>1]'): skuItem = {} skuItem['type'] = "sku" skuItem['from_site'] = self.name size = size_opt.xpath('./@value').extract()[0].strip() skuItem['size'] = size sizes.append(size) skuItem['id'] = color_name + size skuItem['show_product_id'] = item['show_product_id'] skuItem['list_price'] = list_pirce skuItem['current_price'] = current_price skuItem['color'] = color_name if size_opt.xpath('./@data-is-oos').extract()[0] == 'true': skuItem['is_outof_stock'] = True skus.append(skuItem) item['colors'] = [color_name] item['sizes'] = sizes item['skus'] = skus product_items = sel.xpath('//div[@id="style-with-slideshow"]/a') if len(product_items) > 0: related_items_id = [] for product_item in product_items: product_id = product_item.xpath('./@href').extract()[0].split('/')[2] related_items_id.append(product_id) if related_items_id: item['related_items_id'] = related_items_id if len(sel.xpath('//a[@id="size_and_fit_link"]/@data-url'))>0: item['size_info'] = self.base_url + sel.xpath('//a[@id="size_and_fit_link"]/@data-url').extract()[0] size_chart_url = 'http://www.fwrd.com' + sel.xpath('//a[@id="size_and_fit_link"]/@data-url').extract()[0] yield Request(size_chart_url, callback=self.parse_size_chart, meta={'item': item}, dont_filter=True) else: item['size_info'] = '' yield item
def handle_parse_item(self, response, item): current_url = get_base_url(response) if current_url == 'http://www.levi.com/US/en_US/error' or current_url == 'http://global.levi.com': return buyJsonStr = "".join( re.findall( r'(var buyStackJSON = .*;[\s\S]*productCodeMaster\s*=[\s\S]*?;)[\s\S]*?</script>', response.body)) context = execjs.compile(''' %s function getBuyStackJSON(){ return buyStackJSON; } function getProductCodeMaster(){ return productCodeMaster; } ''' % buyJsonStr) stackJson = json.loads(context.call('getBuyStackJSON')) productMaster = str(context.call('getProductCodeMaster')) if productMaster not in stackJson['colorids']: return item["show_product_id"] = productMaster if 'name' in stackJson["colorid"][productMaster].keys(): item['desc'] = stackJson["colorid"][productMaster]['name'] item['list_price'] = stackJson["colorid"][productMaster]['price'][0][ 'amount'] if len(stackJson["colorid"][productMaster]['price']) > 1: item['current_price'] = stackJson["colorid"][productMaster][ 'price'][1]['amount'] else: item['current_price'] = item['list_price'] altImgs = stackJson["colorid"][productMaster]['altViews'] altZoomImgs = stackJson["colorid"][productMaster]['altViewsZoomPaths'] imgDir = stackJson["colorid"][productMaster]['imageURL'] images = [] if len(altImgs): imgKey = 0 for imgVal in altImgs: if re.search('alt\d*\-pdp\.jpg', imgVal) > -1: imgDir = re.search(r'(.+' + productMaster[:5] + r').*', imgDir).group(1) imgDir = imgDir + '-' imageItem = ImageItem() if imgVal[:6] == 'detail': imageItem['image'] = imgDir + imgVal + stackJson[ "product"]['imageSizeDetail'] imageItem['thumbnail'] = imgDir + altZoomImgs[ imgKey] + stackJson["product"][ 'imageSizeDetailThumbnail'] else: imageItem['image'] = imgDir + imgVal + stackJson[ "product"]['imageSizeHero'] imageItem['thumbnail'] = imgDir + altZoomImgs[ imgKey] + stackJson["product"]['imageSizeThumbnail'] images.append(imageItem) imgKey += 1 colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = productMaster colorItem['images'] = images colorItem['name'] = stackJson["colorid"][productMaster]['finish'][ 'title'] colorItem['cover'] = stackJson["colorid"][productMaster]['swatch'] color_name = colorItem['name'] yield colorItem item['skus'] = [] item['dimensions'] = [] item['sizes'] = {} if len(stackJson["attrs"]['waist']) > 0: item['dimensions'].append('waist') item['sizes']['waist'] = stackJson["attrs"]['waist'] if len(stackJson["attrs"]['length']) > 0: item['dimensions'].append('length') item['sizes']['length'] = stackJson["attrs"]['length'] if len(stackJson["attrs"]['size']) > 0: item['dimensions'].append('size') item['sizes']['size'] = stackJson["attrs"]['size'] # #有货的未处理 if stackJson["sku"]: for sku in stackJson["sku"]: if stackJson["sku"][sku]['colorid'] == productMaster: is_continue = False sku_keys = stackJson["sku"][sku].keys() for one_dimension in item['dimensions']: if one_dimension not in sku_keys: is_continue = True break if is_continue: continue skuItem = {} skuItem['type'] = 'sku' skuItem['show_product_id'] = stackJson["sku"][sku][ 'colorid'] skuItem['id'] = stackJson["sku"][sku]['skuid'] skuItem['list_price'] = stackJson["sku"][sku]['price'][0][ 'amount'] if stackJson["sku"][sku]['finalsale']: skuItem['current_price'] = stackJson["sku"][sku][ 'price'][1]['amount'] else: skuItem['current_price'] = skuItem['list_price'] skuItem['color'] = color_name #item['colors'][0] if 'size' in stackJson["sku"][sku]: skuItem['size'] = stackJson["sku"][sku]['size'] elif 'waist' in stackJson["sku"][ sku] and 'length' in stackJson["sku"][sku]: skuItem['size'] = { 'waist': stackJson["sku"][sku]['waist'], 'length': stackJson["sku"][sku]['length'] } elif 'sizedescription' in stackJson["sku"][sku]: skuItem['size'] = stackJson["sku"][sku][ 'sizedescription'] skuItem['from_site'] = 'levi' skuItem['quantity'] = stackJson["sku"][sku]['stock'] skuItem['is_outof_stock'] = False item['skus'].append(skuItem) yield skuItem # print skuItem # print '* ' * 20 item['colors'] = [color_name] sel = Selector(response) item['brand'] = sel.xpath( ".//div[@id='main-container']//meta[@itemprop='brand']/@content" ).extract()[0] size_chart_dom = sel.xpath( ".//div[@class='pdp-sizing']/div[@class='pdp-sizes pdp-size-sizes']/a" ) if len(size_chart_dom.extract()): size_chart = size_chart_dom.xpath("./p/text()").extract()[0] if size_chart == 'Size Chart': item['size_info'] = size_chart_dom.xpath( "./@href").extract()[0] yield item
def handle_parse_item(self, response, item): sel = Selector(response) if len( sel.xpath( ".//div[@class='field mb20 mt20 detail-info-outStock']"). extract()) > 0 or len( sel.xpath( ".//div[@class='soldOut color-red bold h4']")) > 0: return if len(sel.xpath(".//span[@itemprop='sku']/text()")) == 0: return if 'show_product_id' not in item.keys(): item['show_product_id'] = sel.xpath( ".//span[@itemprop='sku']/text()").extract()[0] item['desc'] = sel.xpath( ".//div[@data-tstid='Content_Composition&Care']/dl").extract()[0] item['dimensions'] = ['size', 'color'] if len( sel.xpath(".//a[@data-target='.sizeGuideModal']/@href"). extract()) > 0: item['size_info'] = self.base_url + sel.xpath( ".//a[@data-target='.sizeGuideModal']/@href").extract()[0] product_json = re.search( 'window\.universal_variable\.product[\s]*=[\s]*(.+)[\s]*</script>', response.body).group(1) product_json = unicode(product_json, errors='replace') if len(product_json) > 0: context = execjs.compile(''' var skus = %s; function getDetail(){ return skus; } ''' % product_json) goods_detail = context.call('getDetail') categoryId = goods_detail['categoryId'] designerId = goods_detail['manufacturerId'] productId = goods_detail['id'] storeId = goods_detail['storeId'] if goods_detail['color'] == "": color_name = "One Color" else: color_name = goods_detail['color'] list_price = goods_detail['unit_price'] current_price = goods_detail['unit_sale_price'] else: return images = [] image_lis = sel.xpath( './/ul[@class="sliderProduct js-sliderProduct js-sliderProductPage"]/li' ) for image_li in image_lis: imageItem = ImageItem() imageItem['image'] = image_li.xpath( './a/img/@data-zoom-image').extract()[0] imageItem['thumbnail'] = image_li.xpath( './a/img/@data-large').extract()[0] images.append(imageItem) if images == []: return color = Color() color['type'] = 'color' color['from_site'] = self.name color['show_product_id'] = item['show_product_id'] color['images'] = images color['name'] = color_name cover = re.sub('_480\.jpg', '_70.jpg', images[0]['thumbnail']) color['cover'] = cover yield color # data={'categoryId': str(categoryId), 'designerId': str(designerId), 'productId': str(productId), 'storeId': str(storeId)} # print data # {'storeId': '9446', 'categoryId': '135967', 'designerId': '4981', 'productId': '11497594'} # # url = 'https://us-il.proxymesh.com:31280' # # username = '******' # # password = '******' # # password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() # # auth_handler = urllib2.HTTPBasicAuthHandler(password_mgr) # # opener = urllib2.build_opener(auth_handler) # # # urllib2.install_opener(opener) # # # # # opener = urllib2.build_opener() # # opener.addheaders.append(('Cookie', 'ckm-ctx-sf=/')) # # sku_resp = opener.open('http://www.farfetch.com/product/GetDetailState/', urllib.urlencode(data)) # # sku_resp_str = sku_resp.read() # # sku_detail_str = json.loads(sku_resp_str)['Sizes'].strip() # if len(sku_resp_str) > 0: # context = execjs.compile(''' # var skus = %s; # function getDetail(){ # return skus; # } # ''' % sku_resp_str) # sku_detail = context.call('getDetail') # print sku_detail detail_url = 'http://www.farfetch.com/product/GetDetailState?' + 'categoryId=' + str( categoryId) + '&designerId=' + str( designerId) + '&productId=' + str( productId) + '&storeId' + str(storeId) yield Request(detail_url, callback=self.parse_sku, meta={ 'item': item, 'color_name': color_name, 'list_price': list_price, 'current_price': current_price }, cookies={'ckm-ctx-sf': '/'}, dont_filter=True)
def handle_parse_item(self, response, item): sel = Selector(response) if 'show_product_id' not in item.keys(): item['show_product_id'] = sel.xpath( ".//input[@id='baseNo']/@value").extract()[0] retailPrice = re.sub( r'\s', '', sel.xpath(".//span[@class='retailPrice']/a/text()").extract() [0]) find = retailPrice.find('$') item['list_price'] = retailPrice[find + 1:] item['desc'] = sel.xpath( ".//div[@id='overviewSection']/div[1]/*").extract()[0] if len(sel.xpath( ".//div[@class='sizingChart']/table/*").extract()) > 0: item['desc'] += '<div><table>' for d in sel.xpath(".//div[@class='sizingChart']/table/*"): item['desc'] += d.extract() item['desc'] += '</table></div>' item['dimensions'] = [] item['sizes'] = {} item['colors'] = [] propertyDropDown = sel.xpath(".//div[@class='propertyDropDown']") dropDownValue = {} skuKey = {} if len(propertyDropDown.extract()): largeImage = sel.xpath( ".//input[@id='largeImageSrcTemplate']/@value").extract()[0] zoomImage = sel.xpath( ".//input[@id='zoomImageHrefTemplate']/@value").extract()[0] i = 1 for dropDown in propertyDropDown: title = (sel.xpath(".//label[@id='property" + str(i) + "Label']/@title").extract()[0]).lower() skuKey['property' + str(i)] = title vk = {} dom = dropDown.xpath("./select/option[@value != '']") for d in dom: k = d.xpath("./@value").extract()[0] v = d.xpath("./text()").extract()[0] vk[k] = v if title == 'color': imageItem = ImageItem() colorItem = Color() altImages = sel.xpath(".//a[@class='altImage']") prod_images = [] imageItem['thumbnail'] = re.sub( r'_\d+~\d+', '_' + k + '~460', largeImage) imageItem['image'] = re.sub(r'_\d+~\d+', '_' + k + '~1500', zoomImage) colorItem['cover'] = re.sub(r'_\d+~\d+', '_' + k + '~220', imageItem['thumbnail']) prod_images.append(imageItem.copy()) for altImage in altImages: temp_image = altImage.xpath("./@href").extract()[0] temp_thumbnail = altImage.xpath( "./@data-image").extract()[0] for temp_dict in prod_images: t = 0 if temp_dict[ 'thumbnail'] == temp_thumbnail or imageItem[ 'image'] == temp_image: t = -1 break if t != -1: imageItem['image'] = temp_image imageItem['thumbnail'] = temp_thumbnail prod_images.append(imageItem.copy()) colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = item['show_product_id'] colorItem['images'] = prod_images colorItem['name'] = v + '(' + k + ')' item['colors'].append(colorItem['name']) yield colorItem dropDownValue[title] = vk i += 1 if title == 'color': pass else: item['dimensions'].append(title) item['sizes'][title] = vk.values() if len(item['colors']) == 0: return if len(item['dimensions']) == 0: item['dimensions'] = ['size'] if 'size' in item['sizes'].keys(): pass else: item['sizes']['size'] = ['onesize'] item['cover'] = imageItem['thumbnail'].replace("460", "220") skusStr = "".join( re.findall(r'[\s]*var skus = new TAFFY\([\s]*(\[.*\])[\s]*\);', response.body, re.S)) if skusStr: context = execjs.compile(''' var skus = %s; function getSkus(){ return skus; } ''' % skusStr) skusDict = context.call('getSkus') item['skus'] = [] for sku in skusDict: skuItem = {} skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['list_price'] = item['list_price'] skuItem['current_price'] = sku['finalPromoPrice'][1:] skuItem['color'] = dropDownValue['color'][ sku['property1']] + '(' + sku['property1'] + ')' if sku['property2'] == '': skuItem['size'] = {'size': 'onesize'} skuItem['id'] = sku['property1'] else: skuItem['size'] = { skuKey['property2']: dropDownValue[skuKey['property2']][sku['property2']] } skuItem['id'] = sku['property1'] + '*' + sku['property2'] if sku['property3'] != '': skuItem['size'][skuKey['property3']] = dropDownValue[ skuKey['property3']][sku['property3']] skuItem['id'] += '*' + sku['property3'] skuItem['from_site'] = item['from_site'] skuItem['is_outof_stock'] = False if 'size' in skuItem.keys(): if 'size' in skuItem['size'].keys(): pass else: skuItem['size']['size'] = 'onesize' else: skuItem['size'] = {'size': 'onesize'} item['skus'].append(skuItem) yield item
def handle_parse_item(self, response, baseItem): sel = Selector(response) # bread_crumbs=sel.xpath('//div[@id="divBreadCrumb"]/span') # baseItem['product_type']= # baseItem['category']=bread_crumbs[2].xpath('./a/text()').extract()[0] # import pdb;pdb.set_trace() product_id_re_result = re.findall(r'dtmProductId = [^;]+', response.body) if product_id_re_result and len(product_id_re_result) > 0: product_id_str = product_id_re_result[0] product_id = re.findall(r'\d+', product_id_str)[0] baseItem['show_product_id'] = int(product_id) #baseItem['sub_category']=bread_crumbs[3].xpath('./a/text()').extract()[0] baseItem['type'] = 'base' item_in_stock = sel.xpath('//div[@id="divAvailablity"]') if len(item_in_stock) > 0: baseItem['title'] = sel.xpath( '//div[@id="divCaption"]/h1/text()').extract()[0] desc_a = sel.xpath( '//table[@id="TblProdForkPromo"]//td[@class="contenttd"]' ).extract() desc_b = sel.xpath( '//table[@id="TblProdForkWarnings"]//td[@class="contenttd"]' ).extract() if len(desc_a) > 0: baseItem['desc'] = desc_a[0] if len(desc_b) > 0: baseItem['desc'] = desc_b[0] baseItem['colors'] = ['onecolor'] baseItem['sizes'] = ['onesize'] baseItem['dimensions'] = ['size', 'color'] baseItem['from_site'] = self.name imageItem = ImageItem() images = [] imageItem['thumbnail'] = sel.xpath( '//div[@id="divPImage"]//img/@src').extract()[0] imageItem['image'] = re.sub(r'300(\.\w+)', '500\\1', imageItem['thumbnail']) images.append(imageItem) colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = self.name colorItem['show_product_id'] = baseItem['show_product_id'] colorItem['images'] = images colorItem['cover'] = imageItem['thumbnail'] colorItem['name'] = 'onecolor' # import pdb;pdb.set_trace() yield colorItem skus = [] skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = baseItem['show_product_id'] skuItem['from_site'] = self.name list_price = sel.xpath('//span[@id="rowMSRP"]') if len(list_price) > 0: skuItem['list_price'] = list_price.xpath( './s/text()').extract()[0] skuItem['current_price'] = sel.xpath( '//div[@id="productprice"]/span/text()').extract()[0] baseItem['list_price'] = skuItem['list_price'] baseItem['current_price'] = skuItem['current_price'] else: skuItem['current_price'] = sel.xpath( '//div[@id="productprice"]/span/text()').extract()[0] skuItem['list_price'] = skuItem['current_price'] baseItem['list_price'] = skuItem['list_price'] baseItem['current_price'] = skuItem['current_price'] skuItem['is_outof_stock'] = False skuItem['color'] = 'onecolor' skuItem['id'] = baseItem['show_product_id'] skuItem['size'] = 'onesize' skus.append(skuItem) baseItem['skus'] = skus yield baseItem
def handle_parse_item(self, response, item): if re.match(r'^http:\/\/us\.asos\.com\/mp_sp\/',response.url): sel = Selector(response) url = sel.xpath('//li[@id="mp_li_cnti"]/a/@href').extract()[0] yield Request(url, callback=self.parse_item, cookies={'asos': 'currencyid=1'}, meta={'item': item}) else: skus=[] sel=Selector(response) json_info = re.search("view\(\'(.+\})\'\,", response.body) if not json_info: return else: json_info = json_info.group(1) json_info = "".join(json_info) json_info = json_info.decode("string-escape") goods_detail = json.loads(json_info) descs = sel.xpath('//div[@class="overflow-container"]/div/div') item['desc'] = '' for desc in descs: item['desc'] = item['desc'] + desc.extract() item['title'] = goods_detail['name'] if 'brandName' not in goods_detail.keys(): item['brand'] = 'asos' else: item['brand'] = goods_detail['brandName'] item['from_site'] = self.name if 'price' not in goods_detail.keys(): return item['current_price'] = goods_detail['price']['current'] if float(goods_detail['price']['previous']) != 0: item['list_price'] = goods_detail['price']['previous'] elif float(goods_detail['price']['rrp']) != 0: item['list_price'] = goods_detail['price']['rrp'] else: item['list_price'] = goods_detail['price']['current'] item['show_product_id'] = goods_detail['id'] sizes = [] colors = [] for sku in goods_detail['variants']: skuItem = SkuItem() skuItem['type'] = "sku" skuItem['from_site'] = self.name skuItem['is_outof_stock'] = False skuItem['id'] = sku['variantId'] skuItem['show_product_id'] = goods_detail['id'] skuItem['current_price'] = item['current_price'] skuItem['list_price'] = item['list_price'] skuItem['size'] = sku['size'] if sku['size'] not in sizes: sizes.append(sku['size']) skuItem['color'] = sku['colour'] if sku['colour'] not in colors: colors.append(sku['colour']) skus.append(skuItem) for color_name in colors: images = [] for image in goods_detail['images']: if image['colour'] == '' or (image['colour'] and color_name and len(image['colour']) == len(color_name) and (len(color_name) - difflib.SequenceMatcher(None,color_name,image['colour']).ratio()*len(color_name)) <=1): imageItem = ImageItem() imageItem['image'] = image['url'] + '?$XXL$' imageItem['thumbnail'] = image['url'] images.append(imageItem) color = Color() color['type'] = 'color' color['from_site'] = self.name color['show_product_id'] = goods_detail['id'] color['images'] = images color['name'] = color_name color['cover'] = images[0]['image'] yield color item['skus'] = skus item['sizes'] = list(set(sizes)) item['dimensions'] = ['size'] item['colors'] = colors related_products_url = 'http://us.asos.com/api/product/catalogue/v2/productgroups/ctl/' + str(item['show_product_id']) + '?store=US&store=US¤cy=USD' yield Request('http://us.asos.com/api/product/catalogue/v2/stockprice?productIds=' + str(goods_detail['id']) + '&store=US¤cy=USD', callback=self.parse_stock, meta={'item': item, 'related_products_url': related_products_url}) # color_size_str="".join(re.findall(r"var\s+arrSzeCol_ctl00_ContentMainPage_ctlSeparateProduct[^<]+", response.body)) # sep_image_str="".join(re.findall(r"var\s+arrSepImage_ctl00_ContentMainPage_ctlSeparateProduct[^<]+", response.body)) # thumb_image_str="".join(re.findall(r"var\s+arrThumbImage_ctl00_ContentMainPage_ctlSeparateProduct[^<]+", response.body)) # if len(color_size_str)>0: # context = execjs.compile(''' # %s # %s # %s # function get_color_size(){ # return arrSzeCol_ctl00_ContentMainPage_ctlSeparateProduct; # } # function get_sep_image(){ # return arrSepImage_ctl00_ContentMainPage_ctlSeparateProduct; # } # function get_thumb_image(){ # return arrThumbImage_ctl00_ContentMainPage_ctlSeparateProduct; # } # ''' % (color_size_str, sep_image_str, thumb_image_str)) # color_sizes = context.call('get_color_size') # sep_image= context.call('get_sep_image') # thumb_images = context.call('get_thumb_image') # #import pdb;pdb.set_trace() # if len(sel.xpath('//div[@id="ctl00_ContentMainPage_ctlSeparateProduct_pnlOutofStock"]').extract()) > 0: # return # # if len(sel.xpath('//span[@id="ctl00_ContentMainPage_ctlSeparateProduct_lblProductTitle"]/text()').extract()) > 0: # item['title']=sel.xpath('//span[@id="ctl00_ContentMainPage_ctlSeparateProduct_lblProductTitle"]/text()').extract()[0] # # data_dic_str = sel.xpath('//script[@id="dataDictionary"]/text()') # # product_data_str=data_dic_str.re(r'^var Product\s*=\s*({.*?});')[0] # product_data=eval(product_data_str) # item['show_product_id']=product_data['ProductIID'] # desc=sel.xpath('//div[@id="ctl00_ContentMainPage_productInfoPanel"]//ul') # if len(desc)>0: # item['desc']=desc.extract()[0] # item['brand']=product_data['ProductBrand'] # item['from_site']=self.name # # '''有严重问题,注释掉了''' # # gender_category_str=product_data['ProductCategory'] # # m=re.search(r'(.+)\|(.+)', gender_category_str) # # if m: # # item['gender']=m.group(1).strip() # # m=re.search(r'(.+)\|(.+)', gender_category_str) # # if m: # # item['category']=m.group(2).strip() # # sku_data_str = data_dic_str.re(r'var ProductChildSkuInfo\s*=\s*({.*?});')[0] # sku_data=eval(sku_data_str) # sku_data_list=sku_data['ChildSkuInfo'][item['show_product_id']] # #color_list=sel.xpath('//select[@id="ctl00_ContentMainPage_ctlSeparateProduct_drpdwnColour"]').extract() # if color_sizes: # '''handle color and image''' # # # thumbnail_lis=sel.xpath('//ul[@class="productThumbnails"]//li//img/@src') # # image_lis=sel.xpath('//div[@id="productImages"]//img/@src') # # if len(thumbnail_lis)>0: # # for i in range(len(thumbnail_lis)): # # imageItem=ImageItem() # # imageItem['image']=image_lis[i].extract() # # imageItem['thumbnail']=thumbnail_lis[i].extract() # # images.append(imageItem) # #left three imageItem # images=[] # for thumb_image in thumb_images: # imageItem=ImageItem() # imageItem['image']=thumb_image[2] # imageItem['thumbnail']=thumb_image[0] # images.append(imageItem) # # item_color_names=[] # #all color names of item # # sep_image_dict = {} # for sep_image_arr in sep_image: # key = sep_image_arr[3] # sep_image_dict[key] = {'image': sep_image_arr[2], 'thumbnail': sep_image_arr[0]} # # color_names = sel.xpath('//div[@id="ctl00_ContentMainPage_ctlSeparateProduct_pnlColour"]//option/@value')[1:].extract() # for color_name in color_names: # # lower_color_name = color_name.lower() # if '/' in lower_color_name: # lower_color_name_2 = lower_color_name.replace('/', '') # else: # lower_color_name_2 = lower_color_name # if lower_color_name not in sep_image_dict.keys() and lower_color_name_2 not in sep_image_dict.keys(): # return # imageItem=ImageItem() # imageItem['thumbnail']= sep_image_dict[lower_color_name_2]['thumbnail'] # imageItem['image']= sep_image_dict[lower_color_name_2]['image'] # images.insert(0, imageItem) # # import pdb;pdb.set_trace() # color=Color() # color['type'] ='color' # color['from_site'] = self.name # color['show_product_id'] = product_data['ProductIID'] # color['images'] = images # color['name'] = color_name # color['cover'] = sep_image_dict[lower_color_name_2]['thumbnail'] # # yield color # # item_color_names.append(color_name) # '''handle price''' # #list_price_sel=sel.xpath('//span[@id="ctl00_ContentMainPage_ctlSeparateProduct_lblRRP"]') # sizes=[] # for color_size in color_sizes: # size_id = color_size[0] # size = color_size[1] # if not size.strip(): # size = 'onesize' # # if color_size[3] == "False": # continue # # original_color_name = color_size[2] # for color_name in item_color_names: # tmp_color_name = re.sub(r'[^\w]', '', color_name) # # if tmp_color_name == original_color_name: # original_color_name = color_name # # skuItem=SkuItem() # skuItem['type']="sku" # skuItem['from_site']=self.name # skuItem['is_outof_stock']=False # skuItem['id']=sku_data_list[str(size_id)+original_color_name]['Sku'] # #skuItem['id']=color_size[0] # skuItem['show_product_id']=product_data['ProductIID'] # skuItem['current_price']= color_size[5] # # if color_size[6] == color_size[5] and color_size[8] != '0' and color_size[8] != '0.00': # skuItem['list_price']= color_size[8] # else: # skuItem['list_price']= color_size[6] # # sizes.append(size) # skuItem['color'] = original_color_name # skuItem['size'] = size # skus.append(skuItem) # # item['skus']=skus # item['sizes']=list(set(sizes)) # item['dimensions']=['size'] # item['colors'] = item_color_names # size_info = sel.xpath('//a[@id="ctl00_ContentMainPage_SizeGuideButton_SizeGuideLink"]/@href') # if size_info: # item['size_info'] = size_info.extract()[0] # if not re.match(r'^http', size_info.extract()[0]): # item['size_info'] = self.base_url + size_info.extract()[0] # yield item
def handle_parse_item(self, response, item): sel = Selector(response) if len(sel.xpath(".//div[@class='atg_store_noMatchingItem']")) > 0: return info = sel.xpath(".//div[@class='firstContainer row']") item['brand'] = info.xpath("./h1/a[1]/text()").extract()[0] item['show_product_id'] = info.xpath( "./h1/h2/text()").extract()[0].strip() item['title'] = info.xpath("./h1/a[2]/text()").extract()[0] # item['desc'] = info.xpath("./h3/text()").extract()[0] item['colors'] = [] if len(sel.xpath(".//div[@id='tab1_info']")) > 0: if len(sel.xpath(".//div[@id='tab1_info']/div[2]")) > 0: item['desc'] = sel.xpath( ".//div[@id='tab1_info']/div[1]/table").extract( )[0] + sel.xpath( ".//div[@id='tab1_info']/div[2]/table").extract()[0] else: item['desc'] = sel.xpath( ".//div[@id='tab1_info']/div[1]/table").extract()[0] skusStr = "".join( re.findall(r'window.universal_variable =.+\}\}<\/script>', response.body, re.S)) if len(skusStr) > 0: context = execjs.compile(''' var skus = %s; function getSkus(){ return skus; } ''' % skusStr[27:-9]) skusDict = context.call('getSkus') item['list_price'] = skusDict['product']['unit_price'] item['current_price'] = skusDict['product']['unit_sale_price'] images = [] imageDom = sel.xpath(".//ul[@class='alt_imgs col-md-12']/li") colorItem = Color() for dom in imageDom: imageItem = ImageItem() imageItem['image'] = self.base_url + dom.xpath( "./a/@href").extract()[0] imageItem['thumbnail'] = re.sub('XA\.', 'LA.', imageItem['image']) images.append(imageItem.copy()) colorItem['images'] = images colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = item['show_product_id'] colorItem['name'] = u'one color' colorItem['cover'] = self.base_url + sel.xpath( ".//ul[@class='alt_imgs col-md-12']/li[1]/a/img/@src").extract()[0] yield colorItem item['colors'].append(colorItem['name']) item['dimensions'] = ['size'] item['skus'] = [] sku_item_url_list = [] sku_size_list = [] index = 0 sku_items = sel.xpath(".//div[@id='sizeValues']/div") if len(sku_items) > 0: for sku_item in sku_items: sku_size = sku_item.xpath("./@onclick").extract()[0].split( "'")[3] ajax_id = sku_item.xpath("./@onclick").extract()[0].split( "'")[1] if sku_size.find(' ') != -1: sku_size = re.sub(' ', '%20', sku_size) sku_item_url = self.base_url + sel.xpath( ".//form[@id='colorsizerefreshform']/@action" ).extract( )[0] + '&productId=' + ajax_id + '&selectedSize=' + sku_size sku_item_url_list.append(sku_item_url) sku_size_list.append(sku_size) sku_item_url_list.append( sku_item_url ) # only for avoiding indexError in parse_sku_item when loop reach the last size yield Request(sku_item_url_list[0], callback=self.parse_sku_item, meta={ "sku_size_list": sku_size_list, "sku_item_url_list": sku_item_url_list, "item": item, "index": index }) else: skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = skusDict['product']['id'] skuItem['list_price'] = item['list_price'] skuItem['current_price'] = item['current_price'] skuItem['color'] = u'one color' skuItem['size'] = u'one size' skuItem['id'] = skusDict['product']['sku_code'] skuItem['from_site'] = item['from_site'] if skusDict['product']['stock'] == 0: skuItem['is_outof_stock'] = True item['skus'].append(skuItem) item['sizes'] = [u'one size'] yield item
def handle_parse_item(self, response, baseItem): sel = Selector(response) product_id = sel.xpath('//div[@id="productId"]/text()').extract()[0] baseItem['gender'] = 'men' baseItem['type'] = 'base' baseItem['from_site'] = self.name baseItem['show_product_id'] = product_id baseItem['title'] = sel.xpath( '//span[@class="row product-title"]/text()').extract()[0].strip() size_fit_container = sel.xpath('//div[@id="sizeFitContainer"]') if len(size_fit_container) > 0: size_fit = size_fit_container.extract()[0] baseItem['desc'] = '<div>' + sel.xpath( '//div[@itemprop="description"]').extract( )[0] + size_fit + "</div>" else: baseItem['desc'] = sel.xpath( '//div[@itemprop="description"]').extract()[0] baseItem['dimensions'] = ['size', 'color'] skus = [] product_detail_str = "".join( re.findall(r"var\s+productDetail[^;]+", response.body)) if len(product_detail_str) > 0: context = execjs.compile(''' %s function get_product_detail(){ return productDetail; } ''' % (product_detail_str)) product_detail = context.call('get_product_detail') size_js_infos = product_detail['sizes'] size_infos = {} size_values = [] for size_id in size_js_infos: size_infos[size_js_infos[size_id]['sizeCode']] = size_id size_values.append(size_id) list_price = sel.xpath( '//div[@id="productPrices"]//meta[@itemprop="price"]/@content' ).extract()[0] color_price_blocks = sel.xpath( '//div[@id="productPrices"]//div[@class="priceBlock"]') # color_price_mapping = {} # for color_price_block in color_price_blocks: # color_name = color_price_block.xpath( # './span[@class="priceColors"]/text()').extract() # if len(color_name) > 0: # regular_price_span = color_price_block.xpath( # './span[@class="regularPrice"]/text()').extract() # if len(regular_price_span) > 0: # color_price_mapping[color_name[0]] = regular_price_span[0] # else: # color_price_mapping[color_name[0]] = color_price_block.xpath( # './span[@class="salePrice"]/text()').extract()[0] match = re.search(r'productPage\.sellingPrice\=\'([\d\.]+)\';', response.body) if match is None: current_price = list_price else: current_price = match.group(1) image_items = product_detail['colors'] color_names = [] for key in image_items: imageItems = image_items[key]['images'] color_name = image_items[key]['colorName'].strip() + '-' + str(key) color_names.append(color_name) images = [] tmp_images = [] for image_key in imageItems: imageItem = ImageItem() image = imageItems[image_key] imageItem['thumbnail'] = image['thumbnail'] imageItem['image'] = image['zoom'] tmp_images.append((image['index'], imageItem)) tmp_images = sorted(tmp_images, key=lambda x: x[0]) for tmp_tuple in tmp_images: images.append(tmp_tuple[1]) colorItem = Color() colorItem['type'] = 'color' colorItem['show_product_id'] = baseItem['show_product_id'] colorItem['from_site'] = self.name colorItem['cover'] = image_items[key]['swatch'] colorItem['name'] = color_name colorItem['images'] = images yield colorItem sizes = image_items[key]['sizes'] for size in sizes: size_name = size_infos[size] skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['from_site'] = self.name skuItem['color'] = color_name skuItem['show_product_id'] = baseItem['show_product_id'] skuItem['id'] = key + "-" + size skuItem['size'] = size_name skuItem['list_price'] = list_price skuItem['current_price'] = current_price # if len(color_price_mapping) > 0 and color_name in color_price_mapping.keys(): # skuItem['current_price'] = color_price_mapping[ # colorItem['name']] # else: # skuItem['current_price'] = skuItem['list_price'] skuItem['is_outof_stock'] = False skus.append(skuItem) baseItem['sizes'] = size_values baseItem['colors'] = color_names baseItem['skus'] = skus product_items = sel.xpath( '//ul[@id="similarities"]/li[@class="product"]') if len(product_items) > 0: related_items_id = [] for product_item in product_items: product_id = product_item.xpath( './div/div[@class="info"]/img/@data-product-id').extract( )[0] related_items_id.append(product_id) if related_items_id: baseItem['related_items_id'] = related_items_id yield baseItem
def handle_parse_item(self, response, baseItem): product_detail_str="".join(re.findall(r"var\s+productDetail[^;]+", response.body)) if len(product_detail_str)>0: context = execjs.compile(''' %s function get_product_detail(){ return productDetail; } ''' % (product_detail_str)) product_detail = context.call('get_product_detail') sel = Selector(response) product_id = sel.xpath('//div[@id="productId"]/text()').extract()[0] skus = [] baseItem['from_site'] = self.name baseItem['show_product_id'] = product_id size_js_infos = product_detail['sizes'] size_infos = {} size_values = [] for size_id in size_js_infos: size_infos[size_js_infos[size_id]['sizeCode']] = size_id size_values.append(size_id) list_price = sel.xpath('//div[@id="productPrices"]//meta[@itemprop="price"]/@content').extract()[0] color_price_blocks = sel.xpath('//div[@id="productPrices"]//div[@class="priceBlock"]') color_price_mapping = {} for color_price_block in color_price_blocks: color_name = color_price_block.xpath('./span[@class="priceColors"]/text()').extract() if len(color_name) > 0: regular_price_span = color_price_block.xpath('./span[@class="regularPrice"]/text()').extract() if len(regular_price_span) > 0: color_price_mapping[color_name[0]] = regular_price_span[0] else: color_price_mapping[color_name[0]] = color_price_block.xpath('./span[@class="salePrice"]/text()').extract()[0] image_items = product_detail['colors'] color_names = [] for key in image_items: imageItems = image_items[key]['images'] color_name = image_items[key]['colorName'].strip() color_names.append(color_name) images=[] tmp_images = [] for image_key in imageItems: imageItem = ImageItem() image = imageItems[image_key] imageItem['thumbnail'] = image['thumbnail'] imageItem['image'] = image['zoom'] tmp_images.append((image['index'], imageItem)) tmp_images = sorted(tmp_images, key=lambda x:x[0]) for tmp_tuple in tmp_images: images.append(tmp_tuple[1]) colorItem = Color() colorItem['type'] = 'color' colorItem['show_product_id'] = product_id colorItem['from_site'] = self.name colorItem['cover'] = image_items[key]['swatch'] colorItem['name'] = color_name colorItem['images'] = images yield colorItem sizes = image_items[key]['sizes'] for size in sizes: size_name = size_infos[size] skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['from_site'] = self.name skuItem['color'] = color_name skuItem['show_product_id'] = product_id skuItem['id'] = key+"-"+size skuItem['size'] = size_name skuItem['list_price'] = list_price if len(color_price_mapping)>0 and color_name in color_price_mapping.keys(): # skuItem['current_price'] = sale_price_span.re(r'\d+.?\d*')[0] skuItem['current_price'] = color_price_mapping[colorItem['name']] else: skuItem['current_price'] = skuItem['list_price'] skuItem['is_outof_stock'] = False skus.append(skuItem) baseItem['sizes'] = size_values baseItem['colors']= color_names baseItem['skus'] = skus size_fit_container = sel.xpath('//div[@id="sizeFitContainer"]') if len(size_fit_container)>0: size_fit = size_fit_container.extract()[0] baseItem['desc'] = '<div>'+sel.xpath('//div[@itemprop="description"]').extract()[0]+size_fit+"</div>" else: baseItem['desc'] = sel.xpath('//div[@itemprop="description"]').extract()[0] baseItem['dimensions'] = ['size', 'color'] yield baseItem
def handle_parse_item(self, response, item): sel = Selector(response) if len(sel.xpath('//h1[@class="pageTitle"]/text()')) > 0: return if len(sel.xpath('//section[@class="productDesc"]')) > 0: return """单品s""" if len( sel.xpath( '//div[@class="productQuickDetails"]//p[@class="productPrice sale textSale"]/span/text()' )) > 0: item['current_price'] = sel.xpath( '//div[@class="productQuickDetails"]//p[@class="productPrice sale textSale"]/span/text()' ).extract()[0] item['list_price'] = sel.xpath( '//div[@class="productQuickDetails"]//p[@class="productPrice regular"]/span/text()' ).extract()[0] else: item['current_price'] = sel.xpath( '//div[@class="productQuickDetails"]//p[@class="productPrice "]/span/text()' ).extract()[0] item['list_price'] = sel.xpath( '//div[@class="productQuickDetails"]//p[@class="productPrice "]/span/text()' ).extract()[0] if '-' in item['list_price']: item['list_price'] = item['list_price'].split('-')[-1].strip() if '-' in item['current_price']: item['current_price'] = item['current_price'].split( '-')[-1].strip() item['show_product_id'] = sel.xpath( '//div[@class="productNumber"]/text()').extract()[0] if 'Item' in item['show_product_id']: item['show_product_id'] = re.search( '(\d+)', item['show_product_id']).group(1) item['desc'] = sel.xpath( ".//div[@class='productShortDescription']/p/text()").extract() if len(sel.xpath('//div[@class="longDescription"]')) > 0: item['desc'] = item['desc'] + sel.xpath( '//div[@class="longDescription"]').extract() if len(sel.xpath('div[@class="productSpecs"]')) > 0: item['desc'] = item['desc'] + sel.xpath( 'div[@class="productSpecs"]').extract() item['desc'] = ''.join(item['desc']) variants_json_str = re.search('var variantsJson = (.+);', response.body).group(1) variants_json = json.loads(variants_json_str) item['skus'] = [] item['sizes'] = [] if not variants_json['items']: item['sizes'] = ['One Size'] skuItem = {} skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['id'] = re.search('productId: \"(\d+)\"', response.body).group(1) skuItem['color'] = 'One Color' skuItem['size'] = 'One Size' skuItem['from_site'] = self.name skuItem['list_price'] = item['list_price'] skuItem['current_price'] = item['current_price'] skuItem['is_outof_stock'] = False item['skus'].append(skuItem) else: for variant_json in variants_json['items']: item['sizes'].append(variant_json['id']) skuItem = {} skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['id'] = variant_json['catEntryId'] skuItem['color'] = 'One Color' skuItem['size'] = variant_json['id'] skuItem['from_site'] = self.name skuItem['list_price'] = item['list_price'] skuItem['current_price'] = variant_json['price'] skuItem['is_outof_stock'] = not variant_json['buyable'] item['skus'].append(skuItem) if 'sizeChart' in variants_json['variants'][0].keys(): item['size_chart'] = 'https:' + variants_json['variants'][0][ 'sizeChart'] img_as = sel.xpath(".//a[@class='productThumb thumbnailImage']") images = [] if len(img_as) > 0: for img_a in img_as: imageItem = ImageItem() imageItem['image'] = img_a.xpath("./img/@src").extract()[0] imageItem['thumbnail'] = img_a.xpath("./img/@src").extract()[0] if '?' in imageItem['image']: imageItem['image'] = imageItem['image'].split( '?')[0] + self.image_url_suffix if '?' in imageItem['thumbnail']: imageItem['thumbnail'] = imageItem['thumbnail'].split( '?')[0] + self.cover_url_suffix images.append(imageItem) colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = self.name colorItem['show_product_id'] = item['show_product_id'] colorItem['name'] = 'One Color' colorItem['cover'] = images[0]['thumbnail'] colorItem['images'] = images yield colorItem item['colors'] = ['One Color'] if 'size_chart' in item.keys(): yield Request(item['size_chart'], callback=self.parse_size_chart, meta={"item": item}, dont_filter=True) else: yield item
def handle_parse_item(self, response, item): skus = [] sel = Selector(response) item['from_site'] = self.name if 'whoops' in response.url: logging.warning('anti scraping: ' + response.url) match = re.search(r'"product_id":\s*\[\s*"([^"]+)"\s*\]', response.body) if match is None: return temp_show_product_id = match.group(1) current_price = sel.xpath( '//span[contains(@class, "price-sales")]/text()').extract() if len(current_price) > 0: current_price = current_price[0] list_price = sel.xpath( '//span[contains(@class, "price-standard")]/text()').re( r'(\S+)') if len(list_price) > 0: list_price = list_price[0] else: list_price = current_price item['brand'] = self.name # item['desc']=".".join(sel.xpath('//div[contains(@class, "additional")]/ul/li/text()').extract()) desc1 = sel.xpath( '//div[@class="categorylisting detail"]/div/div').extract()[0] desc2 = sel.xpath( '//div[@class="categorylisting fabric"]/div/div').extract()[0] item['desc'] = re.sub(r'[\t\n]', '', desc1 + desc2) item['desc'] = re.sub('<img.+?>', '', item['desc']) if sel.xpath( '//div[contains(@class, "quantity clearfix")]//p[contains(@class, "in-stock-msg")]/text()' ): colors = [] item_colors_links = sel.xpath( '//div[@id="product-content"]//ul[contains(@class, "swatches color")]//li[contains(@class,"selected")]/a' ) item_sizes = sel.xpath( '//div[@id="product-content"]//div[contains(@class, "value")]//ul[contains(@class, "swatches size")]/li[@class!="emptyswatch unselectable"]//@title' ).extract() item['sizes'] = item_sizes item['dimensions'] = ['size'] item['product_type'] = 'mother-baby' if len(item_colors_links) == 0: item_colors_links = ['one_color'] for item_color_link in item_colors_links: images = [] thumbnails = sel.xpath( '//div[@id="thumbnails"]//li[@class!="thumb pdpvideo"]' ) if thumbnails: for li in thumbnails: #thumbnails_evl=li.xpath('./a/img/@src').extract()[0] imageItem = ImageItem() image_url = li.xpath('./a/img/@src').extract()[0] imageItem['image'] = self.handle_image_url( image_url.encode('utf-8'), 1000, 1000) imageItem['thumbnail'] = self.handle_image_url( image_url.encode('utf-8'), 350, 350) images.append(imageItem) elif sel.xpath( '//div[@id="thumbnails"]/li[@class="thumb pdpvideo"]/a/img/@src' ): imageItem = ImageItem() image_url = sel.xpath( '//img[@class="primary-image"]/@src').extract()[0] imageItem['image'] = self.handle_image_url( image_url.encode('utf-8'), 1000, 1000) imageItem['thumbnail'] = self.handle_image_url( image_url.encode('utf-8'), 350, 350) images.append(imageItem) else: imageItem = ImageItem() image_url = sel.xpath( '//img[@class="primary-image"]/@src').extract()[0] imageItem['image'] = self.handle_image_url( image_url.encode('utf-8'), 1000, 1000) imageItem['thumbnail'] = self.handle_image_url( image_url.encode('utf-8'), 350, 350) images.append(imageItem) if len(item_colors_links) > 0: color_name = item_color_link.xpath( './@title').extract()[0] color_cover = item_color_link.xpath('./@style').re( 'http://[^\)]+')[0] else: color_name = 'one_color' color_cover = images[0]['thumbnail'] colors.append(color_name) show_product_id = temp_show_product_id + "*" + color_name item['show_product_id'] = show_product_id color = Color() color['type'] = 'color' color['from_site'] = self.name color['show_product_id'] = show_product_id color['images'] = images color['name'] = color_name color['cover'] = color_cover yield color for item_size in item_sizes: skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = show_product_id skuItem[ 'id'] = item['show_product_id'] + "*" + item_size skuItem['current_price'] = current_price skuItem['list_price'] = list_price if len(item_colors_links) > 0: skuItem['color'] = item_color_link.xpath( './@title').extract()[0] else: skuItem['color'] = 'one_color' skuItem['size'] = item_size skuItem['from_site'] = self.name skuItem['is_outof_stock'] = False skuItem['quantity'] = sel.xpath( '//select[contains(@id, "Quantity")]//@value' ).extract()[0] #yield skuItem skus.append(skuItem) item['colors'] = colors item['skus'] = skus yield item
def handle_parse_item(self, response, item): pImgStr = "".join(re.findall(r'(pImgs[^;]+;)+', response.body)) context = execjs.compile(''' %s function getPImgs(){ return pImgs; } ''' % pImgStr) pImgs = context.call('getPImgs') sel = Selector(response) outofstock_result = re.search(r'outOfStock[\s]*=[\s]*([^;]+);', response.body) if outofstock_result and outofstock_result.group(1) == 'true': return stock_json_result = re.search(r'var stockJSON[\s]*=[\s]*([^;]+);', response.body) if stock_json_result: stock_dic = eval(stock_json_result.group(1)) if stock_dic: color_price_dic = eval( re.search(r'colorPrices[\s]*=[\s]*([^;]+);', response.body).group(1)) style_id_dic = eval( re.search(r'styleIds[\s]*=[\s]*([^;]+);', response.body).group(1)) product_gender = eval( re.search(r'productGender[\s]*=[\s]*([^;]+);', response.body).group(1)) zeta_categories = eval( re.search(r'zetaCategories[\s]*=[\s]*([^;]+);', response.body).group(1)) category = eval( re.search(r';[\s]*category[\s]*=[\s]*([^;]+);', response.body).group(1)) sub_category = eval( re.search(r'subCategory[\s]*=[\s]*("[^"]+"[\s]*);', response.body).group(1)) dimension_dic = eval( re.search(r'dimensions[\s]*=[\s]*([^;]+);', response.body).group(1)) #dimToUnitToValJSON = eval(re.search(r'dimToUnitToValJSON[\s]*=[\s]*([^;]+);', response.body).group(1)) dimensionIdToNameJson = eval( re.search(r'dimensionIdToNameJson[\s]*=[\s]*([^;]+);', response.body).group(1)) valueIdToNameJSON = eval( re.search(r'valueIdToNameJSON[\s]*=[\s]*([^;]+);', response.body).group(1)) colorNames = eval( re.search(r'colorNames[\s]*=[\s]*({[^}]+}[\s]*);', response.body).group(1)) if len(zeta_categories) > 0: item['product_type'] = zeta_categories[0].values()[0] if category == item['product_type']: item['category'] = sub_category else: item['category'] = category item['sub_category'] = sub_category else: item['product_type'] = category item['category'] = sub_category if 'gender' in response.meta.keys(): meta_gender = response.meta['gender'] if product_gender.lower == 'unisex': if meta_gender == 'boys' or meta_gender == 'girls': item['gender'] = 'kid-unisex' else: item['gender'] = 'unisex' else: item['gender'] = meta_gender '''跳过描述,过于复杂''' size_info_images = [] desc = sel.xpath( '//div[@id="productDescription"]//div[@itemprop="description"]/ul' ).extract() if len(desc) > 0: item['desc'] = desc[0] size_infos = sel.xpath( '//div[@id="productDescription"]//div[@itemprop="description"]/ul/li/a[@class="popup-570-550"]' ) if len(size_infos) > 0: size_info_images = [] for size_info in size_infos: size_info_image_url = size_info.xpath( '@href').extract()[0] if not re.match(r'^http:\/\/', size_info_image_url): size_info_image_url = self.base_url + size_info_image_url size_info_images.append(size_info_image_url) else: desc_ul = sel.xpath( '//div[@id="prdInfoText"]//span[@class="description summary"]/ul' ).extract() if len(desc_ul) == 0: return item['desc'] = desc_ul[0] size_infos = sel.xpath( '//div[@id="prdInfoText"]//span[@class="description summary"]/ul/li/a[@class="popup-570-550"]' ) if len(size_infos) > 0: size_info_images = [] for size_info in size_infos: size_info_image_url = size_info.xpath( '@href').extract()[0] if not re.match(r'^http:\/\/', size_info_image_url): size_info_image_url = self.base_url + size_info_image_url size_info_images.append(size_info_image_url) if len(size_info_images) > 0: item['size_info'] = {'images': size_info_images} colors = [] '''处理color''' for (color, color_name) in colorNames.items(): colorItem = Color() colorItem['type'] = 'color' colorItem['from_site'] = self.name colorItem['show_product_id'] = item['show_product_id'] colorItem['name'] = color_name colors.append(color_name) styleId = str(style_id_dic[color]) #colorItem['cover'] = sel.xpath('//a[@id="frontrow-'+color+'"]/img/@src').extract()[0] if 'p' in pImgs[styleId]['DETAILED'].keys(): colorItem['cover'] = pImgs[styleId]['DETAILED']['p'] elif 'd' in pImgs[styleId]['DETAILED'].keys(): colorItem['cover'] = pImgs[styleId]['DETAILED']['d'] elif '1' in pImgs[styleId]['MULTIVIEW_THUMBNAILS'].keys(): colorItem['cover'] = pImgs[styleId][ 'MULTIVIEW_THUMBNAILS']['1'] elif '4' in pImgs[styleId]['MULTIVIEW_THUMBNAILS'].keys(): colorItem['cover'] = pImgs[styleId][ 'MULTIVIEW_THUMBNAILS']['4'] elif '5' in pImgs[styleId]['MULTIVIEW_THUMBNAILS'].keys(): colorItem['cover'] = pImgs[styleId][ 'MULTIVIEW_THUMBNAILS']['5'] colorImages = pImgs[styleId] thumbImages = colorImages['MULTIVIEW_THUMBNAILS'] images = colorImages['2x'] if len(images) == 0: images = colorImages['MULTIVIEW'] thumbImages = sorted(thumbImages.iteritems(), key=lambda d: d[0]) images_array = [] for image_tuple in thumbImages: imageItem = ImageItem() if image_tuple[0] in images.keys(): imageItem['image'] = images[image_tuple[0]] imageItem['thumbnail'] = image_tuple[1] if image_tuple[0] == 'p' or image_tuple[0] == 'd': images_array.insert(0, imageItem) else: images_array.append(imageItem) colorItem['images'] = images_array yield colorItem item['colors'] = colors dimensions = [] sizes = {} for dimension in dimension_dic: dimensions.append(dimensionIdToNameJson[dimension]) sizes[dimensionIdToNameJson[dimension]] = [] if len(dimensions) == 0: dimensions = ['size'] if len(sizes) == 0: sizes = {'size': ['onesize']} item['dimensions'] = dimensions '''处理sku库存''' skuCollectionsList = [] for sku_stock in stock_dic: color = sku_stock['color'] if color in color_price_dic.keys(): skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['from_site'] = self.name skuItem['show_product_id'] = item['show_product_id'] skuItem['id'] = sku_stock['id'] skuItem["list_price"] = color_price_dic[color]['wasInt'] skuItem['current_price'] = color_price_dic[color]['nowInt'] skuItem['color'] = colorNames[color] size_demension = {} for demension in dimension_dic: if demension in sku_stock.keys( ) and sku_stock[demension] in valueIdToNameJSON.keys(): size_value = valueIdToNameJSON[ sku_stock[demension]]['value'] size_demension[ dimensionIdToNameJson[demension]] = size_value if not size_value in sizes[ dimensionIdToNameJson[demension]]: sizes[dimensionIdToNameJson[demension]].append( size_value) if len(size_demension) == 0: size_demension = {'size': 'onesize'} skuItem['size'] = size_demension skuItem['quantity'] = sku_stock['onHand'] skuItem['is_outof_stock'] = False skuCollectionsList.append(skuItem) item['skus'] = skuCollectionsList item['sizes'] = sizes item = self.handle_dimension_to_name(response, item, dimensionIdToNameJson) yield item
def handle_parse_item(self, response, item): body_json = json.loads(response.body) goods_detail = body_json['data'] if goods_detail['inStock'] == 0: return item['linkhaitao_url'] = response.url item['cover'] = goods_detail['coverImgUrl'] item['desc'] = goods_detail['content']['description'] if 'product_type_id' in os.environ.keys(): self.product_type_id = os.environ['product_type_id'] if 'category_id' in os.environ.keys(): self.category_id = os.environ['category_id'] if self.product_type_id: item['product_type_id'] = int(self.product_type_id) item['product_type'] = 'linkhaitao_' + str(self.product_type_id) if self.category_id: item['category_id'] = int(self.category_id) item['category'] = 'linkhaitao_' + str(self.category_id) if 'editor_flag' in os.environ.keys(): self.editor_flag = os.environ['editor_flag'] if self.editor_flag: item['editor_flag'] = self.editor_flag if 'gender' in os.environ.keys(): self.gender = os.environ['gender'] if self.gender: item['gender'] = self.gender item['dimensions'] = ['size', 'color'] item['brand'] = goods_detail['brand']['name_en'] item['title'] = goods_detail['name'] item['current_price'] = goods_detail['realPriceOrg'] item['list_price'] = goods_detail['mallPriceOrg'] from_site = ''.join(goods_detail['sellerName']['namecn'].split()).lower() if self.is_chinese_word(from_site): from_site = ''.join(goods_detail['sellerName']['namecn'].split()).lower() if "'" in from_site: from_site = from_site.replace("'", "") if '/' in from_site: from_site = from_site.split('/')[0] item['from_site'] = from_site if item['from_site'] == '6pm' or item['from_site'] == '6pm/6pm': item['from_site'] = 'sixpm' spu_id = re.search('spuid=(.+)&?',response.url) if spu_id: spu_id = spu_id.group(1) else: spu_id = re.search('&spu=(.+)&?',response.url).group(1) item['show_product_id'] = spu_id item['url'] = goods_detail['pageUrl'] if self.editor_flag: item['editor_flag'] = self.editor_flag if not goods_detail['skuInfo']: colorItem = Color() images = [] color_names = [] skus=[] for image in goods_detail['coverImgList']: imageItem = ImageItem() imageItem['image'] = image imageItem['thumbnail'] = image images.append(imageItem.copy()) colorItem['images'] = images colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = item['show_product_id'] color_name = 'One Color' if color_name not in color_names: color_names.append(color_name) colorItem['name'] = color_name colorItem['cover'] = goods_detail['coverImgUrl'] yield colorItem skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['list_price'] = item['list_price'] skuItem['current_price'] = item['current_price'] skuItem['color'] = color_name skuItem['id'] = item['show_product_id'] + 'onecolor' skuItem['from_site'] = item['from_site'] if goods_detail['inStock'] == 0: skuItem['is_outof_stock'] = True skuItem['size'] = 'One Size' skus.append(skuItem) item['sizes'] = ['One Size'] else: skus_info = goods_detail['skuInfo']['style']['skustylelist'] color_names = [] skus = [] sizes = [] dimensions_values = {} for sku_info in skus_info: colorItem = Color() images = [] for image in sku_info['coverImgList']: imageItem = ImageItem() imageItem['image'] = image imageItem['thumbnail'] = image images.append(imageItem.copy()) colorItem['images'] = images colorItem['type'] = 'color' colorItem['from_site'] = item['from_site'] colorItem['show_product_id'] = item['show_product_id'] if sku_info['style']: color_name = sku_info['style'] else: color_name = 'One Color' if color_name not in color_names: color_names.append(color_name) colorItem['name'] = color_name colorItem['cover'] = images[0]['image'] yield colorItem for sku in sku_info['data']: skuItem = SkuItem() skuItem['type'] = 'sku' skuItem['show_product_id'] = item['show_product_id'] skuItem['list_price'] = sku['mallPriceOrg'] skuItem['current_price'] =sku['realPriceOrg'] skuItem['color'] = color_name skuItem['id'] = sku['skuid'] skuItem['from_site'] = item['from_site'] if sku['inStock'] == 0: skuItem['is_outof_stock'] = True if len(sku['attr']) == 0: skuItem['size'] = 'One Size' if skuItem['size'] not in sizes: sizes.append(skuItem['size']) else: skuItem['size'] = {} for attr in sku['attr']: skuItem['size'][attr['attrName'].lower()] = attr['attrValue'] if attr['attrName'].lower() not in item['dimensions']: item['dimensions'].append(attr['attrName'].lower()) if attr['attrName'].lower() not in dimensions_values.keys(): dimensions_values[attr['attrName'].lower()] = [attr['attrValue']] else: dimensions_values[attr['attrName'].lower()].append(attr['attrValue']) if 'size' not in skuItem['size'].keys(): skuItem['size']['size'] = 'One Size' dimensions_values['size'] = 'One Size' skus.append(skuItem) if sizes: item['sizes'] = sizes elif dimensions_values: item['sizes'] = dimensions_values else: return item['skus'] = skus item['colors'] = color_names yield item