def parse_color_item(self, response):
sel = Selector(response)
index = response.meta['index']
color_urls = response.meta['color_urls']
baseItem = response.meta['baseItem']
colorItem = Color()
colorItem['type'] = 'color'
colorItem['show_product_id'] = baseItem['show_product_id']
colorItem['from_site'] = self.name
colorItem['name'] = color_urls[index]['color_name']
colorItem['cover'] = color_urls[index]['color_cover']
images = []
imageItem = ImageItem()
image_url = sel.xpath('//meta[@property="og:image"]/@content').extract()[0]
imageItem['image'] = re.sub(r'wid=\d+&hei=\d+', 'wid=1000&hei=1000', image_url)
imageItem['thumbnail'] = re.sub(r'wid=\d+&hei=\d+', 'wid=50&hei=50', image_url)
images.append(imageItem)
image_url2 = sel.xpath('//div[@id="productSwatch"]/img/@src').extract()[0]
imageItem = ImageItem()
imageItem['image'] = re.sub(r'wid=\d+&hei=\d+', 'wid=1000&hei=1000', image_url2)
imageItem['thumbnail'] = re.sub(r'wid=\d+&hei=\d+', 'wid=50&hei=50', image_url2)
images.append(imageItem)
colorItem['images'] = images
yield colorItem
skus = response.meta['skus']
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = baseItem['show_product_id']
skuItem['from_site'] = self.name
skuItem['current_price'] = sel.xpath('//div[@class="prodprice saleprice"]/p/span[@itemprop="price"]/text()').extract()[0]
if len(sel.xpath('//div[@class="prodprice saleprice"]/p/span[@class="basePrice"]/text()').extract()) > 0:
skuItem['list_price'] = sel.xpath('//div[@class="prodprice saleprice"]/p/span[@class="basePrice"]/text()').extract()[0]
else:
skuItem['list_price'] = skuItem['current_price']
skuItem['is_outof_stock'] = False
skuItem['color'] = color_urls[index]['color_name']
skuItem['size'] = 'one-size'
skuItem['id'] = baseItem['show_product_id']
skus.append(skuItem)
if index + 1 == len(color_urls):
baseItem['skus'] = skus
yield baseItem
else:
yield Request(color_urls[index+1]['url'], callback=self.parse_color_item
, meta={'baseItem': baseItem, 'color_urls': color_urls, 'index': index+1, 'skus': skus})
def parse_image(self, response):
color_item = response.meta['item']
show_product_id = response.meta['show_product_id']
sel = Selector(response)
image_boxs = sel.css('.magicThumbBox .clickStreamTag')
images = []
for image_box in image_boxs:
imageItem = ImageItem()
image = image_box.xpath('@href').extract()[0]
thumbnail = image_box.xpath('//img/@src').extract()[0]
if re.match(r'^//', image):
image = 'https:' + image
if re.match(r'^//', thumbnail):
thumbnail = 'https:' + thumbnail
imageItem['image'] = image
imageItem['thumbnail'] = thumbnail
images.append(imageItem)
if 'cover' not in color_item or 'cover_style' not in color_item:
color_item['cover'] = images[0]['thumbnail']
color_item["images"] = images
color_item['show_product_id'] = show_product_id
yield color_item
def parse_color_sku(self, response):
baseItem = response.meta['baseItem']
images_tmp = response.meta['images']
jsonStr = json.loads(response.body)
colors = []
skus = []
sizes = []
for col in jsonStr['Colors']:
images = []
for img in images_tmp:
imageItem = ImageItem()
imageItem['thumbnail'] = '%s%s_%s_%s.%s' % (
img['base'], col['Code10'], img['thum_size'], img['index'],
img['thum_ext'])
imageItem['image'] = '%s%s_%s_%s.%s' % (
img['base'], col['Code10'], img['img_size'], img['index'],
img['img_ext'])
images.append(imageItem)
color = Color()
color['type'] = 'color'
color['from_site'] = 'thecorner'
color['show_product_id'] = baseItem['show_product_id']
color['images'] = images
color['name'] = col['Description']
color['cover_style'] = '#' + col['Rgb']
#color['cover_style'] = 'background-color: #%s;' % (col['Rgb'])
colors.append(col['Description'])
yield color
for size in jsonStr['ModelColorSizes']:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = baseItem['show_product_id']
skuItem['from_site'] = 'thecorner'
skuItem['id'] = size['Color']['Description'].encode(
"utf-8") + "*" + size['Size']['Description']
skuItem['list_price'] = baseItem['list_price']
skuItem['current_price'] = baseItem['current_price']
skuItem['size'] = size['Size']['Description']
skuItem['color'] = size['Color']['Description']
skuItem['is_outof_stock'] = False
skuItem['quantity'] = size['Quantity']
sizes.append(size['Size']['Description'])
skus.append(skuItem)
baseItem['skus'] = skus
baseItem['colors'] = list(set(colors))
baseItem['sizes'] = list(set(sizes))
yield baseItem
def handle_image_item(self, response):
item = response.meta['item']
sel = Selector(response)
colorIds = response.meta['colorIds']
colorUrls = response.meta['colorUrls']
image_color_dict = {}
for colorId in colorIds:
images = []
single_color_imgages = re.findall(
"[^\.{]*\/zoom_variation_" + colorId + "[^{]*2200\.jpg",
response.body)
if len(single_color_imgages) == 0:
single_color_imgages = re.findall(
"[^\.{]*\/zoom_variation_" + colorIds[0] +
"[^{]*2200\.jpg", response.body)
for i in range(len(single_color_imgages)):
imageItem = ImageItem()
imageItem[
'image'] = self.data_url + single_color_imgages[i][1:]
tempImage = re.sub('zoom_variation_', 'main_variation_',
imageItem['image'])
imageItem['thumbnail'] = re.sub('2192x2200', '548x550',
tempImage)
images.append(imageItem.copy())
image_color_dict[colorId] = images
index = 0
yield Request(colorUrls[index],
callback=self.handle_color_item,
meta={
'image_color_dict': image_color_dict,
'colorUrls': colorUrls,
'index': index,
'item': item
})
def parse_img(self, response):
sel = Selector(response)
color = response.meta['color']
images_coloum = sel.xpath(
'//div[contains(@id, "rl_pdd_cover_slider")]//ul[contains(@class, "box")]//li'
)
images = []
for images_col in images_coloum:
imageItem = ImageItem()
imageItem['image'] = images_col.xpath('./img/@src').extract()[0]
imageItem['thumbnail'] = images_col.xpath(
'./img/@small').extract()[0]
images.append(imageItem)
color['images'] = images
yield color
def handle_parse_item(self, response, baseItem):
sel = Selector(response)
baseItem['dimensions'] = ['size', 'color']
baseItem['desc'] = sel.xpath(
'//div[@id="aDescriptionBody"]').extract()[0]
baseItem['brand'] = sel.xpath(
'//span[@itemprop="brand"]/text()').extract()[0]
skus = []
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = baseItem['show_product_id']
skuItem['from_site'] = self.name
skuItem['current_price'] = baseItem['current_price']
skuItem['list_price'] = baseItem['list_price']
skuItem['is_outof_stock'] = False
skuItem['color'] = 'one-color'
skuItem['size'] = 'one-size'
skuItem['id'] = baseItem['show_product_id']
skus.append(skuItem)
imageItem = ImageItem()
imageItem['image'] = 'http:' + sel.xpath(
'//img[@id="ImageUrl"]/@src').extract()[0]
imageItem['thumbnail'] = imageItem['image']
images = []
images.append(imageItem)
colorItem = Color()
colorItem['type'] = 'color'
colorItem['show_product_id'] = baseItem['show_product_id']
colorItem['from_site'] = self.name
colorItem['images'] = images
colorItem['name'] = 'one-color'
yield colorItem
baseItem['skus'] = skus
yield baseItem
def handle_parse_item(self, response, item):
sel = Selector(response)
catalogId = sel.xpath(
'//input[contains(@id, "catalogId")]/@value').extract()[0]
storeId = sel.xpath(
'//input[contains(@id, "storeId")]/@value').extract()[0]
show_product_id = sel.xpath(
'//input[contains(@id, "productId")]/@value').extract()[0]
#baseItem = response.meta['baseItem']
baseItem = item
baseItem['from_site'] = self.name
baseItem['type'] = 'base'
baseItem['title'] = sel.xpath(
'//div[contains(@class, "pdd_title box")]//h3/text()').extract()[0]
baseItem['show_product_id'] = show_product_id
baseItem['desc'] = sel.xpath(
'//div[contains(@class, "pdd_desc pdd_sub_item box")]').extract(
)[0]
baseItem['list_price'] = sel.xpath(
'//p[contains(@class, "pdd_price")]//span/text()').extract()[0]
if sel.xpath('//span[contains(@class, "promo_price")]'):
baseItem['current_price'] = sel.xpath(
'//span[contains(@class, "promo_price")]/text()').extract()[0]
else:
baseItem['current_price'] = baseItem['list_price']
colors = []
skus = []
colorNames = []
sizes = {'size': ['onesize']}
color_coloum = sel.xpath(
'//ul[contains(@class, "pdd_colors_list box")]//li')
images_coloum = sel.xpath(
'//div[contains(@id, "rl_pdd_cover_slider")]//ul[contains(@class, "box")]//li'
)
for colors_col in color_coloum:
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = show_product_id
color['cover'] = colors_col.xpath('./a/img/@src').extract()[0]
color['name'] = colors_col.xpath('./a/img/@alt').extract()[0]
colorNames.append(color['name'])
if colors_col.xpath(
'./a[contains(@class, "pdd_color pdd_color_picked")]//span/@onclick'
):
images = []
for images_col in images_coloum:
imageItem = ImageItem()
imageItem['image'] = images_col.xpath(
'./img/@src').extract()[0]
imageItem['thumbnail'] = images_col.xpath(
'./img/@small').extract()[0]
images.append(imageItem)
color['images'] = images
yield color
else:
#print re.findall(r'\(.*?\)', clickParam)
clickParam = colors_col.xpath(
'./a[contains(@class, "pdd_color")]//span/@onclick'
).extract()[0]
clickParams = re.findall(r"'(.*)'", clickParam)[0].split(',')
MFPARTNUMBER = clickParams[5].replace("'", "")
imgUrl = '%s/webapp/wcs/stores/servlet/ProductDetailFullImageView?catalogId=%s&langId=-1&storeId=%s&MFPARTNUMBER=%s' % (
self.base_url, catalogId, storeId, MFPARTNUMBER)
yield Request(imgUrl.encode('UTF-8'),
meta={'color': color},
callback=self.parse_img)
'''
quantityUrl = 'http://www.ralphlauren.asia/webapp/wcs/stores/servlet/ProductDetailQuantityView?catalogId=12551&langId=-1&storeId=12151'
formdata = {
'SKUId': str(clickParam[2]),
'objectId':'',
'requesttype':'ajax'
}
yield FormRequest(url=sizeUrl, formdata=formdata, callback=self.parse_quantity, meta={ '': '' } )
sizeUrl = self.base_url + '/webapp/wcs/stores/servlet/ProductDetailSizeSelectView?catalogId=12551&langId=-1&storeId=12151'
formdata = {
'Id': str(clickParam[1]),
'SKUId': str(clickParam[2]),
'Color': 'Lime',
'ColorId': str(clickParam[5]),
'Size': '',
'InItemSppSplitChar':'@@',
'objectId':'',
'requesttype':'ajax'
}
yield FormRequest(url=sizeUrl, formdata=formdata, callback=self.parse_size, meta={ 'color': color } )
'''
###
sku_coloum = sel.xpath(
'//select[contains(@id, "rl_pdd_size")]//option ')
for sku_col in sku_coloum:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = show_product_id
skuItem['from_site'] = self.name
skuItem['list_price'] = baseItem['list_price']
skuItem['current_price'] = baseItem['current_price']
skuItem['is_outof_stock'] = False
skuItem['id'] = sel.xpath(
'//input[contains(@id, "selectSKUId")]/@value').extract()[0]
skuItem['color'] = sel.xpath(
'//span[contains(@class, "pdd_current_color")]/text()'
).extract()[0]
#skuItem['size'] = sku_col.xpath('./option/text()').extract()[0]
skuItem['quantity'] = sel.xpath(
'//select[contains(@id, "rl_pdd_qty")]//option/@value'
).extract()[0]
skus.append(skuItem)
baseItem['colors'] = colorNames
baseItem['sizes'] = sizes
baseItem['skus'] = skus
baseItem['dimensions'] = ""
baseItem['brand'] = 'ralph lauren'
baseItem['category'] = sel.xpath(
'//div[contains(@class, "bread bread_bar")]//a[4]/text()').extract(
)[0]
baseItem['product_type'] = sel.xpath(
'//div[contains(@class, "bread bread_bar")]//a[3]/text()').extract(
)[0]
yield baseItem
def handle_parse_item(self, response, item):
sel = Selector(response)
item['show_product_id'] = str(
re.search('productID: \"(\d+)\"', response.body).group(1)).strip()
item['brand'] = str(
re.search('productBrand: \"(.+)\"',
response.body).group(1)).strip()
item['title'] = sel.xpath(
'.//h1[@data-track="product-title"]/text()').extract()[0].strip()
item['desc'] = ''.join(
sel.xpath('//div[@itemprop="description"]/p').extract()).strip()
item['current_price'] = sel.xpath(
'//span[@class="price"]/text()').extract()[0].strip()
list_price_search = re.search('rrp: .+\&\#36\;([\d\.]+).+',
response.body)
if list_price_search:
item['list_price'] = list_price_search.group(1)
else:
item['list_price'] = item['current_price']
images = []
image_divs = sel.xpath(
'//div[@class="product-thumb-box productImageZoom__thumbnailContainer "]'
)
if not image_divs:
return
for image_div in image_divs:
imageItem = ImageItem()
imageItem['thumbnail'] = image_div.xpath('./img/@src').extract()[0]
imageItem['image'] = image_div.xpath(
'./parent::*/@href').extract()[0]
images.append(imageItem)
color_names = sel.xpath(
'//select[@id="opts-2"]/option[position()>1]/text()').extract()
if len(color_names) > 1:
return
if not color_names:
color_names = ['One Color']
item['colors'] = color_names
color = Color()
color['type'] = 'color'
color['from_site'] = item['from_site']
color['show_product_id'] = item['show_product_id']
color['images'] = images
color['name'] = color_names[0]
color['cover'] = images[0]['thumbnail']
yield color
skus = []
sizes = sel.xpath(
'//select[@id="opts-1"]/option[position()>1]/text()').extract()
if not sizes:
sizes = ['One Size']
item['sizes'] = sizes
for size in sizes:
for color_name in color_names:
skuItem = SkuItem()
skuItem['type'] = "sku"
skuItem['from_site'] = item['from_site']
skuItem['id'] = item[
'show_product_id'] + '-' + color_name + '-' + size
skuItem['show_product_id'] = item['show_product_id']
skuItem['current_price'] = item['current_price']
skuItem['list_price'] = item['list_price']
skuItem['size'] = size
skuItem['color'] = color_name
skus.append(skuItem)
item['skus'] = skus
item['dimensions'] = ['size']
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
if len(sel.xpath('//form[@id="product-form"]//meta').extract()) > 1:
return
if len(sel.xpath('//div[@class="sold-out-details"]')) > 0:
return
item['show_product_id'] = sel.xpath(
'//div[@class="product-code"]/span/text()').extract()[0].strip()
imgs = sel.xpath(
'//div[@class="container-imagery"]//ul[@class="thumbnails no-carousel"]/li/img/@src'
).extract()
if len(imgs) == 0:
imgs = sel.xpath(
'//div[@class="container-imagery"]//ul[@class="swiper-wrapper"]/li/img/@src'
).extract()
images = []
for img in imgs:
if 'http:' not in img:
img = 'http:' + img
if 'xs.jpg' in img:
img = img.replace('xs.jpg', 'pp.jpg')
imageItem = ImageItem()
imageItem['image'] = img
imageItem['thumbnail'] = img.replace('pp.jpg', 'm.jpg')
images.append(imageItem)
colorItem = Color()
colorItem['images'] = images
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = 'One Color'
colorItem['cover'] = images[0]['image'].replace('pp.jpg', 'xs.jpg')
# colorItem['cover'] = images[0]['image'].split('_')[0] + '_sw.jpg'
# print colorItem['cover']
# req = requests.get(colorItem['cover'])
# if not req.ok:
# colorItem['cover'] = images[0]['image'].replace('pp.jpg', 'xs.jpg')
yield colorItem
price = int(
sel.xpath('//form[@id="product-form"]/meta/@data-price-full').
extract()[0]) / 100
if len(sel.xpath('//select-dropdown[@class="sku"]/@options')) > 0:
sku_str = sel.xpath(
'//select-dropdown[@class="sku"]/@options').extract()[0]
skus = json.loads(sku_str)
item['skus'] = []
sizes = []
for sku in skus:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = price
skuItem['current_price'] = price
skuItem['color'] = 'One Color'
skuItem['size'] = sku['data']['size']
sizes.append(sku['data']['size'])
skuItem['id'] = sku['id']
skuItem['from_site'] = item['from_site']
if sku['stockLevel'] == 'In_Stock' or sku[
'stockLevel'] == 'Low_Stock':
skuItem['is_outof_stock'] = False
else:
skuItem['is_outof_stock'] = True
item['skus'].append(skuItem)
else:
item['skus'] = []
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = price
skuItem['current_price'] = price
skuItem['color'] = 'One Color'
sizes = ['One Size']
skuItem['size'] = 'One Size'
skuItem['id'] = sel.xpath(
'//input [@class="sku"]/@value').extract()[0]
stock_level = sel.xpath(
'//input [@class="sku"]/@data-stock').extract()[0]
if stock_level == 'In_Stock' or stock_level == 'Low_Stock':
skuItem['is_outof_stock'] = False
else:
skuItem['is_outof_stock'] = True
skuItem['from_site'] = item['from_site']
item['skus'].append(skuItem)
item['gender'] = self.gender
item['colors'] = ['One Color']
item['sizes'] = sizes
item['desc'] = ''
if len(sel.xpath('//widget-show-hide[@id="accordion-1"]//ul/li')) > 0:
item['desc'] = item['desc'] + sel.xpath(
'//widget-show-hide[@id="accordion-1"]//ul/li').extract()[0]
if len(sel.xpath('//widget-show-hide[@id="accordion-2"]//ul/li')) > 0:
item['desc'] = item['desc'] + sel.xpath(
'//widget-show-hide[@id="accordion-2"]//ul/li').extract()[0]
if len(sel.xpath('//widget-show-hide[@id="accordion-2"]//p')) > 0:
item['desc'] = item['desc'] + sel.xpath(
'//widget-show-hide[@id="accordion-2"]//p').extract()[0]
product_items = sel.xpath(
'//widget-show-hide[@name="Editor\'s Notes"]/div[@class="show-hide-content"]/div/p/a'
)
if len(product_items) > 0:
related_items_id = []
for product_item in product_items:
product_id = product_item.xpath('./@href').extract()[0].split(
'/')[-1]
related_items_id.append(product_id)
if related_items_id:
item['related_items_id'] = related_items_id
media_url = 'https://video.net-a-porter.com/videos/productPage/' + item[
'show_product_id'] + '_detail.mp4'
try:
req = requests.head(media_url)
if req.ok:
item['media_url'] = media_url
except Exception as e:
logging.error('error media url: ' + media_url + ' error msg: ' +
str(e))
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
detail_str = re.search('<script>ReactDOM\.render\(React\.createElement\(ProductDesktop\,[\s]*(.+)[\s]*\)\,[\s]*document\.getElementById', response.body)
if detail_str is None:
return
context = execjs.compile('''
var json = %s
function getJson(){
return json;
}
''' % detail_str.group(1))
detail_json = context.call('getJson')
goods_detail = detail_json['initialData']['Model']['StyleModel']
if goods_detail['IsAvailable'] == False:
return
item['desc'] = goods_detail['Description']
item['show_product_id'] = goods_detail['Id']
item['dimensions'] = ['size', 'color']
# if len(sel.xpath('//div[@class="size-chart"]/a/@href'))>0:
# item['size_info'] = sel.xpath('//div[@class="size-chart"]/a/@href').extract()[0]
# else:
item['size_info'] = ''
color_details = goods_detail['StyleMedia']
color_names = []
color_images = {}
image_keys = ['Zoom', 'MaxLarge', 'Large', 'Medium']
for color_detail in color_details:
if color_detail['ColorName'] == '':
continue
if color_detail['ColorName'] not in color_images.keys():
color_images[color_detail['ColorName']]=[]
imageItem = ImageItem()
# imageItem['image'] = color_detail['ImageMediaUri']['Gigantic']
# if color_detail['ImageMediaUri']['Gigantic'] == "":
for image_key in image_keys:
imageItem['image'] = color_detail['ImageMediaUri'][image_key]
if not color_detail['ImageMediaUri'][image_key] or 'g.nordstromimage.com' in color_detail['ImageMediaUri'][image_key]:
continue
else:
imageItem['image'] = color_detail['ImageMediaUri'][image_key]
break
imageItem['thumbnail'] = color_detail['ImageMediaUri']['Large']
if imageItem['thumbnail'] or 'g.nordstromimage.com' in imageItem['thumbnail']:
imageItem['thumbnail'] = color_detail['ImageMediaUri']['Medium']
color_images[color_detail['ColorName']].append(imageItem)
if len(color_images) > 0:
for color_name, images in color_images.items():
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = item['show_product_id']
color['images'] = images
color['name'] = color_name
color['cover'] = images[0]['thumbnail']
yield color
else:
for color_detail in color_details:
imageItem = ImageItem()
for image_key in image_keys:
imageItem['image'] = color_detail['ImageMediaUri'][image_key]
if color_detail['ImageMediaUri'][image_key] == "" or 'g.nordstromimage.com' in color_detail['ImageMediaUri'][image_key]:
continue
else:
imageItem['image'] = color_detail['ImageMediaUri'][image_key]
break
imageItem['thumbnail'] = color_detail['ImageMediaUri']['Large']
if imageItem['thumbnail'] or 'g.nordstromimage.com' in imageItem['thumbnail']:
imageItem['thumbnail'] = color_detail['ImageMediaUri']['Medium']
images = [imageItem]
color_as = sel.xpath('//ul[@class="swatch-list"]/li/a/span/img')
color_names = []
for color_a in color_as:
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = item['show_product_id']
color_name = color_a.xpath('./@alt').extract()[0]
if 'swatch image' in color_name:
color_name = color_name.replace('swatch image','')
color['name'] = color_name.strip()
color_names.append(color_name)
color['images'] = images
color['cover'] = color_a.xpath('./@src').extract()[0]
yield color
list_price_list = []
sizes = []
skus = []
widths = []
for item_group in goods_detail['ChoiceGroups']:
list_price_temp = item_group['Price']['OriginalPrice']
if '–' in list_price_temp:
list_price_temp = list_price_temp.split('–')[1].strip()
list_price_list.append(list_price_temp)
if 'Size' not in item_group.keys() or item_group['Size'] is None:
sizes = ['One Size']
continue
if 'Width' in item_group.keys() and item_group['Width'] is not None:
if 'width' not in item['dimensions']:
item['dimensions'].append('width')
for width_group in item_group['Width']:
if width_group['Value'] not in widths:
widths.append(width_group['Value'])
for size_group in item_group['Size']:
if size_group['Value'] not in sizes:
sizes.append(size_group['Value'])
if widths:
size_width = {'size': sizes, 'width': widths}
else:
size_width = {'size': sizes}
sku_list_price = max(list_price_list)
for sku_detail in goods_detail['Skus']:
skuItem = {}
skuItem['type'] = "sku"
skuItem['from_site'] = self.name
skuItem['id'] = sku_detail['Id']
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = sku_list_price
skuItem['current_price'] = sku_detail['Price']
skuItem['color'] = sku_detail['Color']
if sku_detail['Size']:
skuItem['size'] = {'size': sku_detail['Size']}
if 'Width' in sku_detail.keys() and sku_detail['Width'] is not None:
skuItem['size']['width'] = sku_detail['Width']
else:
skuItem['size'] = {'size': 'One Size'}
skus.append(skuItem)
if len(color_images) > 0:
item['colors'] = color_images.keys()
else:
item['colors'] = color_names
item['sizes'] = size_width
item['skus'] = skus
# if len(sel.xpath('//div[@class="size-chart"]/a/@href'))>0:
# size_chart_url = sel.xpath('//div[@class="size-chart"]/a/@href').extract()[0]
# yield Request(size_chart_url, callback=self.parse_size_chart, meta={'item': item})
# else:
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
item['show_product_id'] = re.search('(\d+)\.html',
response.url).group(1)
item['title'] = sel.xpath(
'//div[@class="product-name"]/h1/text()').extract()[0].strip()
brand = re.search('brand: \'(.+)\'', response.body)
if not brand:
item['brand'] = 'pharmacyonline'
else:
item['brand'] = brand.group(1)
img = re.search('imgUrl: \'(.+)\'', response.body).group(1)
images = []
imageItem = ImageItem()
imageItem[
'thumbnail'] = img + '?imageMogr2/thumbnail/380x380/extent/380x380/background/d2hpdGU='
imageItem['image'] = img
images.append(imageItem)
# item['cover'] = images[0]['thumbnail']
item['colors'] = ['One Color']
color = Color()
color['type'] = 'color'
color['from_site'] = item['from_site']
color['show_product_id'] = item['show_product_id']
color['images'] = images
color['name'] = 'One Color'
color['cover'] = images[0][
'image'] + '?imageMogr2/thumbnail/100x100/extent/100x100/background/d2hpdGU='
yield color
item['desc'] = sel.xpath(
'//div[@class="product-collateral"]').extract()[0]
current_price = sel.xpath(
'//div[@class="DetailNoDis PriceNow last_price_sing"]/span/text()')
if len(current_price) > 0:
item['current_price'] = current_price.extract()[0]
item['list_price'] = item['current_price']
else:
item['current_price'] = sel.xpath(
'//div[@class="DetailPriceContain clearfix"]//div[@class="PriceNow"]/text()'
).extract()[0].strip()
item['list_price'] = sel.xpath(
'//div[@class="DetailPriceContain clearfix"]//p[@class="PriceWas"]/text()'
).extract()[0].strip()
skus = []
item['sizes'] = ['One Size']
skuItem = SkuItem()
skuItem['type'] = "sku"
skuItem['from_site'] = item['from_site']
sku_id = sel.xpath(
'//div[@class="DetailSku"]/text()').extract()[0].strip()
skuItem['id'] = re.search('(\d+)', sku_id).group(1)
skuItem['show_product_id'] = item['show_product_id']
skuItem['current_price'] = item['current_price']
skuItem['list_price'] = item['list_price']
skuItem['size'] = 'One Size'
skuItem['color'] = 'One Color'
skus.append(skuItem)
item['skus'] = skus
item['dimensions'] = ['size']
if len(item['show_product_id']) > 6:
product_id = item['show_product_id'][1:]
else:
product_id = item['show_product_id']
stock_url = 'http://cn.pharmacyonline.com.au/pt_catalog/index/checkQty?product_id=' + product_id
yield Request(stock_url,
callback=self.parse_stock,
meta={"item": item},
dont_filter=True)
def handle_parse_item(self, response, item):
sel = Selector(response)
product_str = re.search('var dataLayer\s*=\s*(.+?);',
response.body).group(1)
product_json = json.loads(product_str)
product = product_json['products'][0]
item['brand'] = 'Tiffany'
item['title'] = product['name']
if not product['price']:
return
item['list_price'] = product['price']
item['current_price'] = product['price']
item['desc'] = sel.xpath(
'//div[@id="drawerDescription"]/div/div').extract()[0]
item['cover'] = sel.xpath('//meta[@property="og:image"]/@content'
).extract()[0] + self.cover_img_surffix
if product['stockStatus'] == 'out of stock':
return
skus = []
sizes = []
if not sel.xpath(
'//select[@id="ctlSkuGroupType1_selItemList"]/option'):
item['show_product_id'] = product['sku']
skuItem = SkuItem()
skuItem['type'] = "sku"
skuItem['from_site'] = item['from_site']
skuItem['size'] = 'One Size'
sizes = ['One Size']
skuItem['color'] = 'One Color'
skuItem['id'] = item['show_product_id'] + '-' + skuItem[
'color'] + '-' + skuItem['size']
skuItem['show_product_id'] = item['show_product_id']
skuItem['current_price'] = item['current_price']
skuItem['list_price'] = item['list_price']
skus.append(skuItem)
else:
item['show_product_id'] = product['groupSku']
size_options = sel.xpath(
'//select[@id="ctlSkuGroupType1_selItemList"]/option')
for size_option in size_options:
skuItem = SkuItem()
skuItem['type'] = "sku"
skuItem['from_site'] = item['from_site']
if not size_option.xpath('./text()').extract():
skuItem['size'] = 'One Size'
else:
skuItem['size'] = size_option.xpath(
'./text()').extract()[0]
sizes.append(skuItem['size'])
skuItem['color'] = 'One Color'
skuItem['id'] = item['show_product_id'] + '-' + skuItem[
'color'] + '-' + skuItem['size']
skuItem['show_product_id'] = item['show_product_id']
skuItem['current_price'] = item['current_price']
skuItem['list_price'] = item['list_price']
skus.append(skuItem)
images = []
imageItem = ImageItem()
imageItem['thumbnail'] = sel.xpath(
'//meta[@property="og:image"]/@content').extract(
)[0] + self.large_img_surffix
imageItem['image'] = sel.xpath('//meta[@property="og:image"]/@content'
).extract()[0] + self.large_img_surffix
images.append(imageItem)
color = Color()
color['type'] = 'color'
color['from_site'] = item['from_site']
color['show_product_id'] = item['show_product_id']
color['images'] = images
color['name'] = 'One Color'
color['cover'] = item['cover']
yield color
item['colors'] = ['One Color']
item['sizes'] = sizes
item['skus'] = skus
item['dimensions'] = ['size']
yield item
def handle_parse_item(self, response, item):
match = re.search(
r'<script type\=\"application\/json\">({"ProductDetails".+?)<\/script>',
response.body)
print match.group(1)
sel = Selector(response)
if match is None:
return
context = execjs.compile('''
var json = %s
function getJson(){
return json;
}
''' % match.group(1))
product_json = context.call('getJson')
main_product = product_json['ProductDetails']['main_products'][0]
item['brand'] = main_product['brand_name']['label']
item['title'] = main_product['short_description']
show_product_id = main_product['product_code']
item['show_product_id'] = show_product_id
item['desc'] = main_product['description']
list_price = main_product['price']['list_price']['usd_currency_value']
if re.findall('\-', list_price):
re.search('([\d\.]+)\s*\-', list_price).group(1)
else:
item['list_price'] = list_price
sale_price = main_product['price']['sale_price']['usd_currency_value']
if re.findall('\-', sale_price):
re.search('([\d\.]+)\s*\-', sale_price).group(1)
else:
item['current_price'] = sale_price
item['dimensions'] = ['size']
skus = []
sizes = {}
sizes['size'] = []
color_names = []
colors = main_product['colors']['colors']
handle_color_map = {}
if len(colors) > 0:
for color in colors:
handle_color_map[color['id']] = color['label']
handle_size_map = {}
if len(main_product['sizes']['sizes']) == 0:
sizes['size'].append('onesize')
else:
for size in main_product['sizes']['sizes']:
handle_size_map[size['id']] = size['value']
sizes['size'].append(size['value'])
image_prefix = 'http:' + main_product['media'][
'images_server_url'] + main_product['media']['images_path']
if len(colors) == 0:
color_name = 'onecolor'
color_names.append(color_name)
common_images = main_product['media']['images']
images = []
for common_image in common_images:
imageItem = ImageItem()
imageItem[
'image'] = image_prefix + common_image + '?wid=970&hei=1293&fmt=jpg'
imageItem[
'thumbnail'] = image_prefix + common_image + '?wid=396&hei=528&fmt=jpg'
images.append(imageItem)
first_thumbnail = images[0]['thumbnail']
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['images'] = images
colorItem['name'] = color_name
colorItem['cover'] = first_thumbnail
colorItem['version'] = '1'
yield colorItem
else:
common_images = main_product['media']['images']
for color in colors:
color_name = color['label']
color_names.append(color_name)
images = []
imageItem = ImageItem()
imageItem['image'] = image_prefix + color[
'colorize_image_url'] + '?wid=970&hei=1293&fmt=jpg'
imageItem['thumbnail'] = image_prefix + color[
'colorize_image_url'] + '?wid=396&hei=528&fmt=jpg'
images.append(imageItem)
first_thumbnail = images[0]['thumbnail']
for common_image in common_images:
imageItem = ImageItem()
imageItem[
'image'] = image_prefix + common_image + '?wid=970&hei=1293&fmt=jpg'
imageItem[
'thumbnail'] = image_prefix + common_image + '?wid=396&hei=528&fmt=jpg'
images.append(imageItem)
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['images'] = images
colorItem['name'] = color_name
colorItem['version'] = '1'
if len(color['value']) > 0:
if re.findall('\#', color['value']):
colorItem['cover_style'] = color['value']
else:
cover_img_str = sel.xpath(
'//li[@class="product-color-options__value" and @data-colorid='
+ str(color["id"]) + ']/@style').extract()
cover_unavi_str = sel.xpath(
'//li[@class="product-color-options__value product-color-options__value--unavailable" and @data-colorid='
+ str(color["id"]) + ']/@style').extract()
cover_sel_str = sel.xpath(
'//li[@class="product-color-options__value product-color-options__value--selected" and @data-colorid='
+ str(color["id"]) + ']/@style').extract()
cover_hid_str = sel.xpath(
'//li[@class="product-color-options__value is-hidden" and @data-colorid='
+ str(color["id"]) + ']/@style').extract()
if len(cover_img_str) > 0:
cover_img = re.search('\((.+)\)',
cover_img_str[0]).group(1)
colorItem['cover'] = 'http:' + cover_img
elif len(cover_unavi_str) > 0:
cover_img_str = cover_unavi_str[0]
cover_img = re.search('\((.+)\)',
cover_img_str).group(1)
colorItem['cover'] = 'http:' + cover_img
elif len(cover_sel_str) > 0:
cover_img_str = cover_sel_str[0]
cover_img = re.search('\((.+)\)',
cover_img_str).group(1)
colorItem['cover'] = 'http:' + cover_img
elif len(cover_hid_str) > 0:
cover_img_str = cover_hid_str[0]
cover_img = re.search('\((.+)\)',
cover_img_str).group(1)
colorItem['cover'] = 'http:' + cover_img
else:
colorItem['cover'] = first_thumbnail
else:
colorItem['cover'] = first_thumbnail
yield colorItem
item['colors'] = color_names
for sku in main_product['skus']['skus']:
sku_id = sku['sku_id']
if sku_id == 'DUMMY':
continue
if sku['color_id'] == -1:
color_name = 'onecolor'
else:
color_name = handle_color_map[sku['color_id']]
if sku['size_id'] == -1:
size = 'onesize'
else:
size = handle_size_map[sku['size_id']]
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['from_site'] = item['from_site']
skuItem['id'] = sku_id
skuItem['size'] = size
skuItem['color'] = color_name
if sku['status_alias'] == 'soldout' or sku[
'status_alias'] == 'waitlist':
skuItem['is_outof_stock'] = True
else:
skuItem['is_outof_stock'] = False
if len(sku['price']['sale_price']['usd_currency_value']) > 0:
skuItem['current_price'] = sku['price']['sale_price'][
'usd_currency_value']
else:
continue
if len(sku['price']['list_price']['usd_currency_value']) > 0:
skuItem['list_price'] = sku['price']['list_price'][
'usd_currency_value']
else:
continue
skus.append(skuItem)
item['sizes'] = sizes
item['skus'] = skus
if main_product['size_guide_link']['enabled'] == True:
sizeInfo = main_product['size_guide_link']['url']
findQ = sizeInfo.find("?")
if findQ != -1:
item['size_info'] = sizeInfo[:findQ]
else:
item['size_info'] = sizeInfo
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
item['current_price'] = product_price[6:]
item['list_price'] = rrp_prices[1][6:]
if 'show_product_id' not in item.keys():
item['show_product_id'] = eval(re.search(r'productID:(.*),', response.body).group(1))
if 'brand' not in item.keys():
item['brand'] = eval(re.search(r'productBrand:(.*),', response.body).group(1))
if item['brand'] == '':
item['brand'] = 'allsole'
desc = sel.xpath(".//div[@itemprop='description']/*").extract()
if len(desc) > 0:
item['desc'] = desc[0]
else:
item['desc'] = ''
sizeDoms = sel.xpath(".//select[@id='opts-1']/option[string-length(@value) > 0]")
item['dimensions'] = ['size']
item['skus'] = []
if len(sizeDoms.extract()):
colorDoms = sel.xpath(".//select[@id='opts-2']/option[@value>0]")
if len(colorDoms.extract()):
color = colorDoms.xpath("./text()").extract()[0]
item["colors"] = [color]
else:
item["colors"] = ["onecolor"]
item['sizes'] = []
for dom in sizeDoms:
curr_size = dom.xpath("./text()").extract()[0]
item['sizes'].append(curr_size)
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = item["colors"][0] + '*' + curr_size
skuItem['color'] = item["colors"][0]
skuItem['size'] = curr_size
##skuItem['size_num'] = re.search(r'\d+', curr_size).group(0)
skuItem['from_site'] = self.name
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
else:
item['sizes'] = ['onesize']
item['colors'] = ['onecolor']
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = item['colors'][0] + '*' + item['sizes'][0]
skuItem['color'] = item['colors'][0]
skuItem['size'] = item['sizes'][0]
skuItem['from_site'] = self.name
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
imgDoms = sel.xpath(".//div[@class='media']//li[@class='n-unit']")
images = []
if len(imgDoms.extract()):
for dom in imgDoms:
imageItem = ImageItem()
imageItem['image'] = dom.xpath("./a/@href").extract()[0]
imageItem['thumbnail'] = dom.xpath("./a/div/img/@src").extract()[0]
images.append(imageItem)
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = self.name
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = item['colors'][0]
colorItem['cover'] = images[0]['thumbnail']
colorItem['images'] = images
yield colorItem
yield item
def handle_parse_item(self, response, item):
if 'Error' in response.url:
return
sel = Selector(response)
item['category'] = sel.xpath(
'//section[@id="Breadcrumbs"]/nav/a[last()]/text()').extract()[0]
show_product_id = sel.xpath('//span[@id="Sku"]/text()').extract()[-1]
details_str = ''.join(
re.search('sf\.productDetail\.init\((\{[\s\S]+?\})\)\;',
response.body).group(1))
context = execjs.compile('''
var skus = %s;
function getSkus(){
return skus;
}
''' % details_str)
item['skus'] = []
details = context.call('getSkus')
skus_json = details['variants']
images_json = details['images']
images = []
sizes = []
width = []
dimensions_dict = {}
is_in_stock = False
for children in details['children']:
if children['c'] == show_product_id:
color_name = children['v']
is_in_stock = True
if not is_in_stock:
return
list_price = ''.join(
sel.xpath(
'//div[@id="Price"]/div[@class="floatLeft productPriceInfo"]/span[@class="origPrice"]//span/text()'
).extract()).strip()
current_price = ''.join(
sel.xpath(
'//div[@id="Price"]/span[@class="floatLeft productPrice "]/text() | //div[@id="Price"]/span[@class="floatLeft productPrice "]/span/text()'
).extract()).strip()
if not list_price:
list_price = ''.join(
sel.xpath(
'//div[@id="Price"]//span/text()').extract()).strip()
current_price = list_price
for sku_json in skus_json:
skuItem = {}
skuItem['type'] = 'sku'
skuItem['id'] = sku_json['v']
skuItem['show_product_id'] = show_product_id
skuItem['list_price'] = list_price
skuItem['current_price'] = current_price
skuItem['from_site'] = item['from_site']
skuItem['is_outof_stock'] = False
skuItem['size'] = {}
for sku in sku_json['a']:
if sku['n'] == 'custom.color':
skuItem['color'] = sku['v']
else:
if sku['n'] not in dimensions_dict.keys():
dimensions_dict[sku['n']] = []
if sku['v'] not in dimensions_dict[sku['n']]:
dimensions_dict[sku['n']].append(sku['v'])
skuItem['size'][sku['n']] = sku['v']
# if sku['n'] == 'size':
# skuItem['size']['size'] = sku['v']
# dimensions_dict['size'].append(sku['v'])
# if sku['n'] == 'width':
# skuItem['size']['width'] = sku['v']
# dimensions_dict['width'].append(sku['v'])
if skuItem['size']['size'] not in sizes:
sizes.append(skuItem['size']['size'])
if 'width' in skuItem['size'].keys(
) and skuItem['size']['width'] not in width:
width.append(skuItem['size']['width'])
item['skus'].append(skuItem)
for img_json in images_json:
if img_json['c'] == show_product_id:
imageItem = ImageItem()
imageItem['thumbnail'] = img_json['f']
imageItem['image'] = img_json[
'f'][:img_json['f'].find('?')] + '?wid=1280&hei=900'
images.append(imageItem)
if not images:
return
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = show_product_id
colorItem['images'] = images
colorItem['cover'] = images[0]['thumbnail']
colorItem['name'] = color_name
yield colorItem
item['show_product_id'] = show_product_id
item['dimensions'] = dimensions_dict.keys()
if len(sel.xpath('//div[@id="Description"]/p/text()')) > 0:
item['desc'] = sel.xpath(
'//div[@id="Description"]/p/text()').extract()[0].strip()
else:
item['desc'] = ''.join(
sel.xpath('//ul[@class="productFeaturesDetails"]//li').extract(
)).strip()
item['colors'] = [color_name]
item['sizes'] = dimensions_dict
yield item
def parse_color(self, response):
item = response.meta['item']
color_forms = response.meta['color_forms']
# print color_forms
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
response_body = re.sub(r'null', 'None', response.body)
color_form_response = eval(response_body)
if 'images' in color_form_response.keys():
thumbnail = color_form_response['images'][0]['name']
if len(color_form_response['images']) < 4:
return
image = color_form_response['images'][3]['name']
if '/600' not in image:
image = re.sub('/[\d]+/[\d]+', '/600/600', image)
color_name = color_form_response['variations'][0]['options'][0][
'name']
cover_style = color_form_response['variations'][0]['options'][0][
'value']
price_str = color_form_response['price']
m = re.search(r'\d+\.\d+', price_str)
if m:
current_price = m.group(0)
imageItem = ImageItem()
imageItem['thumbnail'] = 'https://s4.thcdn.com/' + thumbnail
imageItem['image'] = 'https://s4.thcdn.com/' + image
images = []
images.append(imageItem)
colorItem['name'] = color_name
colorItem['images'] = images
colorItem['cover_style'] = cover_style
yield colorItem
if 'skus' not in item.keys():
item['skus'] = []
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = color_name + '*' + 'onesize'
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = current_price
skuItem['from_site'] = item['from_site']
skuItem['color'] = color_name
skuItem['size'] = 'onesize'
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
if len(color_forms) == 0:
yield item
else:
# import pdb;pdb.set_trace()
color_ajax_url = 'https://www.lookfantastic.com/variations.json?productId=' + str(
item['show_product_id'])
color_form_value = color_forms.pop(0)
# for color_form_value in color_form:
form_data = {
'option1': str(color_form_value),
'selected': '1',
'variation1': '4'
}
# import pdb;pdb.set_trace()
yield FormRequest(color_ajax_url,
formdata=form_data,
callback=self.parse_color,
meta={
'item': item,
'colorItem': colorItem,
'color_forms': color_forms
})
def handle_parse_item(self, response, item):
sel = Selector(response)
soldout_span = sel.xpath(
'//div[@class="cta-container"]//span[contains(@class, "soldout")]/text()'
).extract()
if len(soldout_span) > 0:
return
# siteObjStr = "".join(re.findall(r'(var siteObj = [\s\S]*exitting = false;)', response.body))
# context = execjs.compile('''
# %s
# function getSiteObj(){
# return siteObj;
# }
# ''' % siteObjStr)
# tmpStr = json.dumps(context.call('getSiteObj'))
# siteObj = json.loads(tmpStr)
# product_id = siteObj['productID']
# product_title = siteObj['productTitle']
# product_price = siteObj['productPrice']
# product_brand = siteObj['productBrand']
# product_rrp = siteObj['rrp']
product_id = eval(
re.search(r'productID:(.*),', response.body).group(1))
# product_title = eval(re.search(r'productTitle:(.*),', response.body).group(1))
product_price = eval(
re.search(r'productPrice:(.*),', response.body).group(1))
product_brand = eval(
re.search(r'productBrand:(.*),', response.body).group(1))
product_rrp = eval(re.search(r'rrp:(.*),', response.body).group(1))
if len(sel.xpath(".//h1[@itemprop='name']/text()")) > 0:
product_title = sel.xpath(
".//h1[@itemprop='name']/text()").extract()[0]
else:
current_url = response.url
product_title = ' '.join(current_url.split('/')[3].split('-'))
rrp_dr = re.compile(r'<[^>]+>', re.S)
rrp = rrp_dr.sub('', product_rrp)
rrp_prices = re.split(r': ', rrp)
if len(
sel.xpath(
".//*[@id='health-beauty']//div[@itemprop='description']").
extract()) > 0:
product_desc = sel.xpath(
".//*[@id='health-beauty']//div[@itemprop='description']"
).extract()[0]
else:
product_desc = ''
if product_id:
item['show_product_id'] = product_id
item['title'] = product_title
item['current_price'] = product_price.replace('£', '')
item['list_price'] = rrp_prices[1].replace('£', '')
item['desc'] = product_desc
item['brand'] = product_brand
item['dimensions'] = ['size']
item['sizes'] = ['onesize']
item['colors'] = ['onecolor']
if re.findall(r'men', item['brand']):
item['gender'] = 'men'
else:
return
thumbnails = sel.xpath(
".//ul[@class='product-thumbnails nav-items product-large-view-thumbs productImageZoom__thumbs']/li//div[contains(@class, 'product-thumb-box')]/img/@src"
).extract()
#thumbnails = sel.xpath(".//div[@class='product-thumb-box']/img/@src").extract()
# viewimages = sel.xpath(".//div[@class='product-thumb-box']/parent::*/@href").extract() #480*480
viewimages = sel.xpath(
".//ul[@class='product-thumbnails nav-items product-large-view-thumbs productImageZoom__thumbs']/li/a/@href"
).extract() #600*600
images = []
color_cover = None
if len(thumbnails) > 0:
imgKey = 0
for imgVal in thumbnails:
imageItem = ImageItem()
imageItem['image'] = handle_no_http(viewimages[imgKey].strip())
imageItem['thumbnail'] = handle_no_http(imgVal)
if not color_cover:
color_cover = imageItem['thumbnail']
images.append(imageItem)
imgKey += 1
color_form = sel.xpath(
'//div[@class="variation-dropdowns"]/form//option/@value').extract(
)[1:]
color_names = sel.xpath(
'//div[@class="variation-dropdowns"]/form//option/text()').extract(
)[1:]
size_or_color_str = sel.xpath(
'//div[@class="variation-dropdowns"]/form//label/text()').extract(
)
if len(size_or_color_str) > 0:
size_or_color = size_or_color_str[0]
if len(color_form) > 0 and size_or_color == 'Shade:':
item['colors'] = color_names
color_ajax_url = 'https://www.lookfantastic.com/variations.json?productId=' + str(
item['show_product_id'])
color_form_value = color_form.pop(0)
# for color_form_value in color_form:
form_data = {
'option1': str(color_form_value),
'selected': '1',
'variation1': '4'
}
# import pdb;pdb.set_trace()
yield FormRequest(color_ajax_url,
formdata=form_data,
callback=self.parse_color,
meta={
'item': item,
'color_forms': color_form
})
elif len(color_form) > 0 and size_or_color == 'Size:':
item['sizes'] = color_names
#pop color
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = 'onecolor'
colorItem['images'] = images
colorItem['cover'] = color_cover
yield colorItem
size_ajax_url = 'https://www.lookfantastic.com/variations.json?productId=' + str(
item['show_product_id'])
color_form_value = color_form.pop(0)
# for color_form_value in color_form:
form_data = {
'option1': str(color_form_value),
'selected': '1',
'variation1': '1'
}
# import pdb;pdb.set_trace()
yield FormRequest(size_ajax_url,
formdata=form_data,
callback=self.parse_size,
meta={
'item': item,
'color_forms': color_form
})
else:
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = product_id
colorItem['images'] = images
colorItem['cover'] = color_cover
colorItem['name'] = 'onecolor'
yield colorItem
item['skus'] = []
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = product_id
skuItem['id'] = product_id
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['from_site'] = item['from_site']
skuItem['color'] = 'onecolor'
skuItem['size'] = 'onesize'
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
# yield skuItem
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
if len(sel.xpath('//div[@id="sold-out-div"]')) > 0:
return
if len(sel.xpath('//div[@class="price_box sale"]'))>0:
list_pirce = sel.xpath('//div[@class="price_box sale"]/span[@class="price"]/text()').extract()[0].strip()
current_price = sel.xpath('//div[@class="price_box sale"]/span[@class="discount_price"]/text()').extract()[0].strip()
print 1111111111
elif len(sel.xpath('//div[@class="price_box"]/span[@class="discount_price"]/text()')) > 0:
list_pirce = sel.xpath('//div[@class="price_box"]/span[@class="discount_price"]/text()').extract()[0].strip()
current_price = list_pirce
print 2222222222222
elif len(sel.xpath('//div[@id="tr-pdp-price--sale"]')) > 0:
list_pirce = sel.xpath('//div[@id="tr-pdp-price--sale"]/span[@class="prices__retail-strikethrough u-margin-l--lg"]/text()').extract()[0]
current_price = sel.xpath('//div[@id="tr-pdp-price--sale"]/span[@class="prices__markdown"]/text()').extract()[0]
print 33333333333333
elif len(sel.xpath('//div[@class="product_info"]/div[@class="eagle"]/div[@class="prices"]/span/text()'))>0:
list_pirce = sel.xpath('//div[@class="product_info"]/div[@class="eagle"]/div[@class="prices"]/span/text()').extract()[0].strip()
print list_pirce
current_price = list_pirce
print 444444444444444444
else:
return
if 'USD' in list_pirce:
list_pirce = list_pirce.replace('USD', '')
if 'USD' in current_price:
current_price = current_price.replace('USD', '')
item['show_product_id'] = sel.xpath('//span[@id="find-your-size"]/a/@data-code').extract()[0]
item['desc'] = sel.xpath('//div[@id="details"]/ul').extract()[0]
item['dimensions'] = ['size', 'color']
images = []
image_divs = sel.xpath('.//div[@class="product-detail-image-zoom"]')
for image_div in image_divs:
imageItem = ImageItem()
imageItem['image'] = image_div.xpath('./img/@src').extract()[0]
imageItem['thumbnail'] = re.sub('fw/z', 'fw/p', imageItem['image'])
images.append(imageItem)
if not images:
return
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = item['show_product_id']
color['images'] = images
if len(sel.xpath('//select[@id="color-select"]'))>0:
color_name = sel.xpath('//select[@id="color-select"]/option[1]/@value').extract()[0].strip()
color['name'] = color_name
else:
color_name_ext = sel.xpath('//div[@class="color_dd"]/div[@class="title one_sizeonly"]/text()').extract()
if color_name_ext:
color_name = color_name_ext[0].strip()
else:
color_name = 'One Color'
color['name'] = color_name
cover = re.sub('fw/z', 'fw/c', images[0]['thumbnail'])
color['cover'] = cover
yield color
skus = []
sizes = []
if len(sel.xpath('//div[@class="size_dd"]/div[@class="title one_sizeonly"]'))>0:
skuItem = {}
skuItem['type'] = "sku"
skuItem['from_site'] = self.name
size = sel.xpath('//div[@class="size_dd"]/div[@class="title one_sizeonly"]/text()').extract()[0].strip()
skuItem['size'] = size
sizes.append(size)
skuItem['id'] = color_name + size
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = list_pirce
skuItem['current_price'] = current_price
skuItem['color'] = color_name
skus.append(skuItem)
else:
for size_opt in sel.xpath('//select[@id="size-select"]/option[position()>1]'):
skuItem = {}
skuItem['type'] = "sku"
skuItem['from_site'] = self.name
size = size_opt.xpath('./@value').extract()[0].strip()
skuItem['size'] = size
sizes.append(size)
skuItem['id'] = color_name + size
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = list_pirce
skuItem['current_price'] = current_price
skuItem['color'] = color_name
if size_opt.xpath('./@data-is-oos').extract()[0] == 'true':
skuItem['is_outof_stock'] = True
skus.append(skuItem)
item['colors'] = [color_name]
item['sizes'] = sizes
item['skus'] = skus
product_items = sel.xpath('//div[@id="style-with-slideshow"]/a')
if len(product_items) > 0:
related_items_id = []
for product_item in product_items:
product_id = product_item.xpath('./@href').extract()[0].split('/')[2]
related_items_id.append(product_id)
if related_items_id:
item['related_items_id'] = related_items_id
if len(sel.xpath('//a[@id="size_and_fit_link"]/@data-url'))>0:
item['size_info'] = self.base_url + sel.xpath('//a[@id="size_and_fit_link"]/@data-url').extract()[0]
size_chart_url = 'http://www.fwrd.com' + sel.xpath('//a[@id="size_and_fit_link"]/@data-url').extract()[0]
yield Request(size_chart_url, callback=self.parse_size_chart, meta={'item': item}, dont_filter=True)
else:
item['size_info'] = ''
yield item
def handle_parse_item(self, response, item):
current_url = get_base_url(response)
if current_url == 'http://www.levi.com/US/en_US/error' or current_url == 'http://global.levi.com':
return
buyJsonStr = "".join(
re.findall(
r'(var buyStackJSON = .*;[\s\S]*productCodeMaster\s*=[\s\S]*?;)[\s\S]*?</script>',
response.body))
context = execjs.compile('''
%s
function getBuyStackJSON(){
return buyStackJSON;
}
function getProductCodeMaster(){
return productCodeMaster;
}
''' % buyJsonStr)
stackJson = json.loads(context.call('getBuyStackJSON'))
productMaster = str(context.call('getProductCodeMaster'))
if productMaster not in stackJson['colorids']:
return
item["show_product_id"] = productMaster
if 'name' in stackJson["colorid"][productMaster].keys():
item['desc'] = stackJson["colorid"][productMaster]['name']
item['list_price'] = stackJson["colorid"][productMaster]['price'][0][
'amount']
if len(stackJson["colorid"][productMaster]['price']) > 1:
item['current_price'] = stackJson["colorid"][productMaster][
'price'][1]['amount']
else:
item['current_price'] = item['list_price']
altImgs = stackJson["colorid"][productMaster]['altViews']
altZoomImgs = stackJson["colorid"][productMaster]['altViewsZoomPaths']
imgDir = stackJson["colorid"][productMaster]['imageURL']
images = []
if len(altImgs):
imgKey = 0
for imgVal in altImgs:
if re.search('alt\d*\-pdp\.jpg', imgVal) > -1:
imgDir = re.search(r'(.+' + productMaster[:5] + r').*',
imgDir).group(1)
imgDir = imgDir + '-'
imageItem = ImageItem()
if imgVal[:6] == 'detail':
imageItem['image'] = imgDir + imgVal + stackJson[
"product"]['imageSizeDetail']
imageItem['thumbnail'] = imgDir + altZoomImgs[
imgKey] + stackJson["product"][
'imageSizeDetailThumbnail']
else:
imageItem['image'] = imgDir + imgVal + stackJson[
"product"]['imageSizeHero']
imageItem['thumbnail'] = imgDir + altZoomImgs[
imgKey] + stackJson["product"]['imageSizeThumbnail']
images.append(imageItem)
imgKey += 1
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = productMaster
colorItem['images'] = images
colorItem['name'] = stackJson["colorid"][productMaster]['finish'][
'title']
colorItem['cover'] = stackJson["colorid"][productMaster]['swatch']
color_name = colorItem['name']
yield colorItem
item['skus'] = []
item['dimensions'] = []
item['sizes'] = {}
if len(stackJson["attrs"]['waist']) > 0:
item['dimensions'].append('waist')
item['sizes']['waist'] = stackJson["attrs"]['waist']
if len(stackJson["attrs"]['length']) > 0:
item['dimensions'].append('length')
item['sizes']['length'] = stackJson["attrs"]['length']
if len(stackJson["attrs"]['size']) > 0:
item['dimensions'].append('size')
item['sizes']['size'] = stackJson["attrs"]['size']
# #有货的未处理
if stackJson["sku"]:
for sku in stackJson["sku"]:
if stackJson["sku"][sku]['colorid'] == productMaster:
is_continue = False
sku_keys = stackJson["sku"][sku].keys()
for one_dimension in item['dimensions']:
if one_dimension not in sku_keys:
is_continue = True
break
if is_continue:
continue
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = stackJson["sku"][sku][
'colorid']
skuItem['id'] = stackJson["sku"][sku]['skuid']
skuItem['list_price'] = stackJson["sku"][sku]['price'][0][
'amount']
if stackJson["sku"][sku]['finalsale']:
skuItem['current_price'] = stackJson["sku"][sku][
'price'][1]['amount']
else:
skuItem['current_price'] = skuItem['list_price']
skuItem['color'] = color_name #item['colors'][0]
if 'size' in stackJson["sku"][sku]:
skuItem['size'] = stackJson["sku"][sku]['size']
elif 'waist' in stackJson["sku"][
sku] and 'length' in stackJson["sku"][sku]:
skuItem['size'] = {
'waist': stackJson["sku"][sku]['waist'],
'length': stackJson["sku"][sku]['length']
}
elif 'sizedescription' in stackJson["sku"][sku]:
skuItem['size'] = stackJson["sku"][sku][
'sizedescription']
skuItem['from_site'] = 'levi'
skuItem['quantity'] = stackJson["sku"][sku]['stock']
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
yield skuItem
# print skuItem
# print '* ' * 20
item['colors'] = [color_name]
sel = Selector(response)
item['brand'] = sel.xpath(
".//div[@id='main-container']//meta[@itemprop='brand']/@content"
).extract()[0]
size_chart_dom = sel.xpath(
".//div[@class='pdp-sizing']/div[@class='pdp-sizes pdp-size-sizes']/a"
)
if len(size_chart_dom.extract()):
size_chart = size_chart_dom.xpath("./p/text()").extract()[0]
if size_chart == 'Size Chart':
item['size_info'] = size_chart_dom.xpath(
"./@href").extract()[0]
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
if len(
sel.xpath(
".//div[@class='field mb20 mt20 detail-info-outStock']").
extract()) > 0 or len(
sel.xpath(
".//div[@class='soldOut color-red bold h4']")) > 0:
return
if len(sel.xpath(".//span[@itemprop='sku']/text()")) == 0:
return
if 'show_product_id' not in item.keys():
item['show_product_id'] = sel.xpath(
".//span[@itemprop='sku']/text()").extract()[0]
item['desc'] = sel.xpath(
".//div[@data-tstid='Content_Composition&Care']/dl").extract()[0]
item['dimensions'] = ['size', 'color']
if len(
sel.xpath(".//a[@data-target='.sizeGuideModal']/@href").
extract()) > 0:
item['size_info'] = self.base_url + sel.xpath(
".//a[@data-target='.sizeGuideModal']/@href").extract()[0]
product_json = re.search(
'window\.universal_variable\.product[\s]*=[\s]*(.+)[\s]*</script>',
response.body).group(1)
product_json = unicode(product_json, errors='replace')
if len(product_json) > 0:
context = execjs.compile('''
var skus = %s;
function getDetail(){
return skus;
}
''' % product_json)
goods_detail = context.call('getDetail')
categoryId = goods_detail['categoryId']
designerId = goods_detail['manufacturerId']
productId = goods_detail['id']
storeId = goods_detail['storeId']
if goods_detail['color'] == "":
color_name = "One Color"
else:
color_name = goods_detail['color']
list_price = goods_detail['unit_price']
current_price = goods_detail['unit_sale_price']
else:
return
images = []
image_lis = sel.xpath(
'.//ul[@class="sliderProduct js-sliderProduct js-sliderProductPage"]/li'
)
for image_li in image_lis:
imageItem = ImageItem()
imageItem['image'] = image_li.xpath(
'./a/img/@data-zoom-image').extract()[0]
imageItem['thumbnail'] = image_li.xpath(
'./a/img/@data-large').extract()[0]
images.append(imageItem)
if images == []:
return
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = item['show_product_id']
color['images'] = images
color['name'] = color_name
cover = re.sub('_480\.jpg', '_70.jpg', images[0]['thumbnail'])
color['cover'] = cover
yield color
# data={'categoryId': str(categoryId), 'designerId': str(designerId), 'productId': str(productId), 'storeId': str(storeId)}
# print data
# {'storeId': '9446', 'categoryId': '135967', 'designerId': '4981', 'productId': '11497594'}
# # url = 'https://us-il.proxymesh.com:31280'
# # username = '******'
# # password = '******'
# # password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
# # auth_handler = urllib2.HTTPBasicAuthHandler(password_mgr)
# # opener = urllib2.build_opener(auth_handler)
# # # urllib2.install_opener(opener)
# #
# # # opener = urllib2.build_opener()
# # opener.addheaders.append(('Cookie', 'ckm-ctx-sf=/'))
# # sku_resp = opener.open('http://www.farfetch.com/product/GetDetailState/', urllib.urlencode(data))
# # sku_resp_str = sku_resp.read()
# # sku_detail_str = json.loads(sku_resp_str)['Sizes'].strip()
# if len(sku_resp_str) > 0:
# context = execjs.compile('''
# var skus = %s;
# function getDetail(){
# return skus;
# }
# ''' % sku_resp_str)
# sku_detail = context.call('getDetail')
# print sku_detail
detail_url = 'http://www.farfetch.com/product/GetDetailState?' + 'categoryId=' + str(
categoryId) + '&designerId=' + str(
designerId) + '&productId=' + str(
productId) + '&storeId' + str(storeId)
yield Request(detail_url,
callback=self.parse_sku,
meta={
'item': item,
'color_name': color_name,
'list_price': list_price,
'current_price': current_price
},
cookies={'ckm-ctx-sf': '/'},
dont_filter=True)
def handle_parse_item(self, response, item):
sel = Selector(response)
if 'show_product_id' not in item.keys():
item['show_product_id'] = sel.xpath(
".//input[@id='baseNo']/@value").extract()[0]
retailPrice = re.sub(
r'\s', '',
sel.xpath(".//span[@class='retailPrice']/a/text()").extract()
[0])
find = retailPrice.find('$')
item['list_price'] = retailPrice[find + 1:]
item['desc'] = sel.xpath(
".//div[@id='overviewSection']/div[1]/*").extract()[0]
if len(sel.xpath(
".//div[@class='sizingChart']/table/*").extract()) > 0:
item['desc'] += '<div><table>'
for d in sel.xpath(".//div[@class='sizingChart']/table/*"):
item['desc'] += d.extract()
item['desc'] += '</table></div>'
item['dimensions'] = []
item['sizes'] = {}
item['colors'] = []
propertyDropDown = sel.xpath(".//div[@class='propertyDropDown']")
dropDownValue = {}
skuKey = {}
if len(propertyDropDown.extract()):
largeImage = sel.xpath(
".//input[@id='largeImageSrcTemplate']/@value").extract()[0]
zoomImage = sel.xpath(
".//input[@id='zoomImageHrefTemplate']/@value").extract()[0]
i = 1
for dropDown in propertyDropDown:
title = (sel.xpath(".//label[@id='property" + str(i) +
"Label']/@title").extract()[0]).lower()
skuKey['property' + str(i)] = title
vk = {}
dom = dropDown.xpath("./select/option[@value != '']")
for d in dom:
k = d.xpath("./@value").extract()[0]
v = d.xpath("./text()").extract()[0]
vk[k] = v
if title == 'color':
imageItem = ImageItem()
colorItem = Color()
altImages = sel.xpath(".//a[@class='altImage']")
prod_images = []
imageItem['thumbnail'] = re.sub(
r'_\d+~\d+', '_' + k + '~460', largeImage)
imageItem['image'] = re.sub(r'_\d+~\d+',
'_' + k + '~1500',
zoomImage)
colorItem['cover'] = re.sub(r'_\d+~\d+',
'_' + k + '~220',
imageItem['thumbnail'])
prod_images.append(imageItem.copy())
for altImage in altImages:
temp_image = altImage.xpath("./@href").extract()[0]
temp_thumbnail = altImage.xpath(
"./@data-image").extract()[0]
for temp_dict in prod_images:
t = 0
if temp_dict[
'thumbnail'] == temp_thumbnail or imageItem[
'image'] == temp_image:
t = -1
break
if t != -1:
imageItem['image'] = temp_image
imageItem['thumbnail'] = temp_thumbnail
prod_images.append(imageItem.copy())
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['images'] = prod_images
colorItem['name'] = v + '(' + k + ')'
item['colors'].append(colorItem['name'])
yield colorItem
dropDownValue[title] = vk
i += 1
if title == 'color':
pass
else:
item['dimensions'].append(title)
item['sizes'][title] = vk.values()
if len(item['colors']) == 0:
return
if len(item['dimensions']) == 0:
item['dimensions'] = ['size']
if 'size' in item['sizes'].keys():
pass
else:
item['sizes']['size'] = ['onesize']
item['cover'] = imageItem['thumbnail'].replace("460", "220")
skusStr = "".join(
re.findall(r'[\s]*var skus = new TAFFY\([\s]*(\[.*\])[\s]*\);',
response.body, re.S))
if skusStr:
context = execjs.compile('''
var skus = %s;
function getSkus(){
return skus;
}
''' % skusStr)
skusDict = context.call('getSkus')
item['skus'] = []
for sku in skusDict:
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = sku['finalPromoPrice'][1:]
skuItem['color'] = dropDownValue['color'][
sku['property1']] + '(' + sku['property1'] + ')'
if sku['property2'] == '':
skuItem['size'] = {'size': 'onesize'}
skuItem['id'] = sku['property1']
else:
skuItem['size'] = {
skuKey['property2']:
dropDownValue[skuKey['property2']][sku['property2']]
}
skuItem['id'] = sku['property1'] + '*' + sku['property2']
if sku['property3'] != '':
skuItem['size'][skuKey['property3']] = dropDownValue[
skuKey['property3']][sku['property3']]
skuItem['id'] += '*' + sku['property3']
skuItem['from_site'] = item['from_site']
skuItem['is_outof_stock'] = False
if 'size' in skuItem.keys():
if 'size' in skuItem['size'].keys():
pass
else:
skuItem['size']['size'] = 'onesize'
else:
skuItem['size'] = {'size': 'onesize'}
item['skus'].append(skuItem)
yield item
def handle_parse_item(self, response, baseItem):
sel = Selector(response)
# bread_crumbs=sel.xpath('//div[@id="divBreadCrumb"]/span')
# baseItem['product_type']=
# baseItem['category']=bread_crumbs[2].xpath('./a/text()').extract()[0]
# import pdb;pdb.set_trace()
product_id_re_result = re.findall(r'dtmProductId = [^;]+',
response.body)
if product_id_re_result and len(product_id_re_result) > 0:
product_id_str = product_id_re_result[0]
product_id = re.findall(r'\d+', product_id_str)[0]
baseItem['show_product_id'] = int(product_id)
#baseItem['sub_category']=bread_crumbs[3].xpath('./a/text()').extract()[0]
baseItem['type'] = 'base'
item_in_stock = sel.xpath('//div[@id="divAvailablity"]')
if len(item_in_stock) > 0:
baseItem['title'] = sel.xpath(
'//div[@id="divCaption"]/h1/text()').extract()[0]
desc_a = sel.xpath(
'//table[@id="TblProdForkPromo"]//td[@class="contenttd"]'
).extract()
desc_b = sel.xpath(
'//table[@id="TblProdForkWarnings"]//td[@class="contenttd"]'
).extract()
if len(desc_a) > 0:
baseItem['desc'] = desc_a[0]
if len(desc_b) > 0:
baseItem['desc'] = desc_b[0]
baseItem['colors'] = ['onecolor']
baseItem['sizes'] = ['onesize']
baseItem['dimensions'] = ['size', 'color']
baseItem['from_site'] = self.name
imageItem = ImageItem()
images = []
imageItem['thumbnail'] = sel.xpath(
'//div[@id="divPImage"]//img/@src').extract()[0]
imageItem['image'] = re.sub(r'300(\.\w+)', '500\\1',
imageItem['thumbnail'])
images.append(imageItem)
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = self.name
colorItem['show_product_id'] = baseItem['show_product_id']
colorItem['images'] = images
colorItem['cover'] = imageItem['thumbnail']
colorItem['name'] = 'onecolor'
# import pdb;pdb.set_trace()
yield colorItem
skus = []
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = baseItem['show_product_id']
skuItem['from_site'] = self.name
list_price = sel.xpath('//span[@id="rowMSRP"]')
if len(list_price) > 0:
skuItem['list_price'] = list_price.xpath(
'./s/text()').extract()[0]
skuItem['current_price'] = sel.xpath(
'//div[@id="productprice"]/span/text()').extract()[0]
baseItem['list_price'] = skuItem['list_price']
baseItem['current_price'] = skuItem['current_price']
else:
skuItem['current_price'] = sel.xpath(
'//div[@id="productprice"]/span/text()').extract()[0]
skuItem['list_price'] = skuItem['current_price']
baseItem['list_price'] = skuItem['list_price']
baseItem['current_price'] = skuItem['current_price']
skuItem['is_outof_stock'] = False
skuItem['color'] = 'onecolor'
skuItem['id'] = baseItem['show_product_id']
skuItem['size'] = 'onesize'
skus.append(skuItem)
baseItem['skus'] = skus
yield baseItem
def handle_parse_item(self, response, item):
if re.match(r'^http:\/\/us\.asos\.com\/mp_sp\/',response.url):
sel = Selector(response)
url = sel.xpath('//li[@id="mp_li_cnti"]/a/@href').extract()[0]
yield Request(url, callback=self.parse_item, cookies={'asos': 'currencyid=1'}, meta={'item': item})
else:
skus=[]
sel=Selector(response)
json_info = re.search("view\(\'(.+\})\'\,", response.body)
if not json_info:
return
else:
json_info = json_info.group(1)
json_info = "".join(json_info)
json_info = json_info.decode("string-escape")
goods_detail = json.loads(json_info)
descs = sel.xpath('//div[@class="overflow-container"]/div/div')
item['desc'] = ''
for desc in descs:
item['desc'] = item['desc'] + desc.extract()
item['title'] = goods_detail['name']
if 'brandName' not in goods_detail.keys():
item['brand'] = 'asos'
else:
item['brand'] = goods_detail['brandName']
item['from_site'] = self.name
if 'price' not in goods_detail.keys():
return
item['current_price'] = goods_detail['price']['current']
if float(goods_detail['price']['previous']) != 0:
item['list_price'] = goods_detail['price']['previous']
elif float(goods_detail['price']['rrp']) != 0:
item['list_price'] = goods_detail['price']['rrp']
else:
item['list_price'] = goods_detail['price']['current']
item['show_product_id'] = goods_detail['id']
sizes = []
colors = []
for sku in goods_detail['variants']:
skuItem = SkuItem()
skuItem['type'] = "sku"
skuItem['from_site'] = self.name
skuItem['is_outof_stock'] = False
skuItem['id'] = sku['variantId']
skuItem['show_product_id'] = goods_detail['id']
skuItem['current_price'] = item['current_price']
skuItem['list_price'] = item['list_price']
skuItem['size'] = sku['size']
if sku['size'] not in sizes:
sizes.append(sku['size'])
skuItem['color'] = sku['colour']
if sku['colour'] not in colors:
colors.append(sku['colour'])
skus.append(skuItem)
for color_name in colors:
images = []
for image in goods_detail['images']:
if image['colour'] == '' or (image['colour'] and color_name and len(image['colour']) == len(color_name) and (len(color_name) - difflib.SequenceMatcher(None,color_name,image['colour']).ratio()*len(color_name)) <=1):
imageItem = ImageItem()
imageItem['image'] = image['url'] + '?$XXL$'
imageItem['thumbnail'] = image['url']
images.append(imageItem)
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = goods_detail['id']
color['images'] = images
color['name'] = color_name
color['cover'] = images[0]['image']
yield color
item['skus'] = skus
item['sizes'] = list(set(sizes))
item['dimensions'] = ['size']
item['colors'] = colors
related_products_url = 'http://us.asos.com/api/product/catalogue/v2/productgroups/ctl/' + str(item['show_product_id']) + '?store=US&store=US¤cy=USD'
yield Request('http://us.asos.com/api/product/catalogue/v2/stockprice?productIds=' + str(goods_detail['id']) + '&store=US¤cy=USD', callback=self.parse_stock, meta={'item': item, 'related_products_url': related_products_url})
# color_size_str="".join(re.findall(r"var\s+arrSzeCol_ctl00_ContentMainPage_ctlSeparateProduct[^<]+", response.body))
# sep_image_str="".join(re.findall(r"var\s+arrSepImage_ctl00_ContentMainPage_ctlSeparateProduct[^<]+", response.body))
# thumb_image_str="".join(re.findall(r"var\s+arrThumbImage_ctl00_ContentMainPage_ctlSeparateProduct[^<]+", response.body))
# if len(color_size_str)>0:
# context = execjs.compile('''
# %s
# %s
# %s
# function get_color_size(){
# return arrSzeCol_ctl00_ContentMainPage_ctlSeparateProduct;
# }
# function get_sep_image(){
# return arrSepImage_ctl00_ContentMainPage_ctlSeparateProduct;
# }
# function get_thumb_image(){
# return arrThumbImage_ctl00_ContentMainPage_ctlSeparateProduct;
# }
# ''' % (color_size_str, sep_image_str, thumb_image_str))
# color_sizes = context.call('get_color_size')
# sep_image= context.call('get_sep_image')
# thumb_images = context.call('get_thumb_image')
# #import pdb;pdb.set_trace()
# if len(sel.xpath('//div[@id="ctl00_ContentMainPage_ctlSeparateProduct_pnlOutofStock"]').extract()) > 0:
# return
#
# if len(sel.xpath('//span[@id="ctl00_ContentMainPage_ctlSeparateProduct_lblProductTitle"]/text()').extract()) > 0:
# item['title']=sel.xpath('//span[@id="ctl00_ContentMainPage_ctlSeparateProduct_lblProductTitle"]/text()').extract()[0]
#
# data_dic_str = sel.xpath('//script[@id="dataDictionary"]/text()')
#
# product_data_str=data_dic_str.re(r'^var Product\s*=\s*({.*?});')[0]
# product_data=eval(product_data_str)
# item['show_product_id']=product_data['ProductIID']
# desc=sel.xpath('//div[@id="ctl00_ContentMainPage_productInfoPanel"]//ul')
# if len(desc)>0:
# item['desc']=desc.extract()[0]
# item['brand']=product_data['ProductBrand']
# item['from_site']=self.name
#
# '''有严重问题，注释掉了'''
# # gender_category_str=product_data['ProductCategory']
# # m=re.search(r'(.+)\|(.+)', gender_category_str)
# # if m:
# # item['gender']=m.group(1).strip()
# # m=re.search(r'(.+)\|(.+)', gender_category_str)
# # if m:
# # item['category']=m.group(2).strip()
#
# sku_data_str = data_dic_str.re(r'var ProductChildSkuInfo\s*=\s*({.*?});')[0]
# sku_data=eval(sku_data_str)
# sku_data_list=sku_data['ChildSkuInfo'][item['show_product_id']]
# #color_list=sel.xpath('//select[@id="ctl00_ContentMainPage_ctlSeparateProduct_drpdwnColour"]').extract()
# if color_sizes:
# '''handle color and image'''
#
# # thumbnail_lis=sel.xpath('//ul[@class="productThumbnails"]//li//img/@src')
# # image_lis=sel.xpath('//div[@id="productImages"]//img/@src')
# # if len(thumbnail_lis)>0:
# # for i in range(len(thumbnail_lis)):
# # imageItem=ImageItem()
# # imageItem['image']=image_lis[i].extract()
# # imageItem['thumbnail']=thumbnail_lis[i].extract()
# # images.append(imageItem)
# #left three imageItem
# images=[]
# for thumb_image in thumb_images:
# imageItem=ImageItem()
# imageItem['image']=thumb_image[2]
# imageItem['thumbnail']=thumb_image[0]
# images.append(imageItem)
#
# item_color_names=[]
# #all color names of item
#
# sep_image_dict = {}
# for sep_image_arr in sep_image:
# key = sep_image_arr[3]
# sep_image_dict[key] = {'image': sep_image_arr[2], 'thumbnail': sep_image_arr[0]}
#
# color_names = sel.xpath('//div[@id="ctl00_ContentMainPage_ctlSeparateProduct_pnlColour"]//option/@value')[1:].extract()
# for color_name in color_names:
#
# lower_color_name = color_name.lower()
# if '/' in lower_color_name:
# lower_color_name_2 = lower_color_name.replace('/', '')
# else:
# lower_color_name_2 = lower_color_name
# if lower_color_name not in sep_image_dict.keys() and lower_color_name_2 not in sep_image_dict.keys():
# return
# imageItem=ImageItem()
# imageItem['thumbnail']= sep_image_dict[lower_color_name_2]['thumbnail']
# imageItem['image']= sep_image_dict[lower_color_name_2]['image']
# images.insert(0, imageItem)
# # import pdb;pdb.set_trace()
# color=Color()
# color['type'] ='color'
# color['from_site'] = self.name
# color['show_product_id'] = product_data['ProductIID']
# color['images'] = images
# color['name'] = color_name
# color['cover'] = sep_image_dict[lower_color_name_2]['thumbnail']
#
# yield color
#
# item_color_names.append(color_name)
# '''handle price'''
# #list_price_sel=sel.xpath('//span[@id="ctl00_ContentMainPage_ctlSeparateProduct_lblRRP"]')
# sizes=[]
# for color_size in color_sizes:
# size_id = color_size[0]
# size = color_size[1]
# if not size.strip():
# size = 'onesize'
#
# if color_size[3] == "False":
# continue
#
# original_color_name = color_size[2]
# for color_name in item_color_names:
# tmp_color_name = re.sub(r'[^\w]', '', color_name)
#
# if tmp_color_name == original_color_name:
# original_color_name = color_name
#
# skuItem=SkuItem()
# skuItem['type']="sku"
# skuItem['from_site']=self.name
# skuItem['is_outof_stock']=False
# skuItem['id']=sku_data_list[str(size_id)+original_color_name]['Sku']
# #skuItem['id']=color_size[0]
# skuItem['show_product_id']=product_data['ProductIID']
# skuItem['current_price']= color_size[5]
#
# if color_size[6] == color_size[5] and color_size[8] != '0' and color_size[8] != '0.00':
# skuItem['list_price']= color_size[8]
# else:
# skuItem['list_price']= color_size[6]
#
# sizes.append(size)
# skuItem['color'] = original_color_name
# skuItem['size'] = size
# skus.append(skuItem)
#
# item['skus']=skus
# item['sizes']=list(set(sizes))
# item['dimensions']=['size']
# item['colors'] = item_color_names
# size_info = sel.xpath('//a[@id="ctl00_ContentMainPage_SizeGuideButton_SizeGuideLink"]/@href')
# if size_info:
# item['size_info'] = size_info.extract()[0]
# if not re.match(r'^http', size_info.extract()[0]):
# item['size_info'] = self.base_url + size_info.extract()[0]
# yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
if len(sel.xpath(".//div[@class='atg_store_noMatchingItem']")) > 0:
return
info = sel.xpath(".//div[@class='firstContainer row']")
item['brand'] = info.xpath("./h1/a[1]/text()").extract()[0]
item['show_product_id'] = info.xpath(
"./h1/h2/text()").extract()[0].strip()
item['title'] = info.xpath("./h1/a[2]/text()").extract()[0]
# item['desc'] = info.xpath("./h3/text()").extract()[0]
item['colors'] = []
if len(sel.xpath(".//div[@id='tab1_info']")) > 0:
if len(sel.xpath(".//div[@id='tab1_info']/div[2]")) > 0:
item['desc'] = sel.xpath(
".//div[@id='tab1_info']/div[1]/table").extract(
)[0] + sel.xpath(
".//div[@id='tab1_info']/div[2]/table").extract()[0]
else:
item['desc'] = sel.xpath(
".//div[@id='tab1_info']/div[1]/table").extract()[0]
skusStr = "".join(
re.findall(r'window.universal_variable =.+\}\}<\/script>',
response.body, re.S))
if len(skusStr) > 0:
context = execjs.compile('''
var skus = %s;
function getSkus(){
return skus;
}
''' % skusStr[27:-9])
skusDict = context.call('getSkus')
item['list_price'] = skusDict['product']['unit_price']
item['current_price'] = skusDict['product']['unit_sale_price']
images = []
imageDom = sel.xpath(".//ul[@class='alt_imgs col-md-12']/li")
colorItem = Color()
for dom in imageDom:
imageItem = ImageItem()
imageItem['image'] = self.base_url + dom.xpath(
"./a/@href").extract()[0]
imageItem['thumbnail'] = re.sub('XA\.', 'LA.', imageItem['image'])
images.append(imageItem.copy())
colorItem['images'] = images
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = u'one color'
colorItem['cover'] = self.base_url + sel.xpath(
".//ul[@class='alt_imgs col-md-12']/li[1]/a/img/@src").extract()[0]
yield colorItem
item['colors'].append(colorItem['name'])
item['dimensions'] = ['size']
item['skus'] = []
sku_item_url_list = []
sku_size_list = []
index = 0
sku_items = sel.xpath(".//div[@id='sizeValues']/div")
if len(sku_items) > 0:
for sku_item in sku_items:
sku_size = sku_item.xpath("./@onclick").extract()[0].split(
"'")[3]
ajax_id = sku_item.xpath("./@onclick").extract()[0].split(
"'")[1]
if sku_size.find(' ') != -1:
sku_size = re.sub(' ', '%20', sku_size)
sku_item_url = self.base_url + sel.xpath(
".//form[@id='colorsizerefreshform']/@action"
).extract(
)[0] + '&productId=' + ajax_id + '&selectedSize=' + sku_size
sku_item_url_list.append(sku_item_url)
sku_size_list.append(sku_size)
sku_item_url_list.append(
sku_item_url
) # only for avoiding indexError in parse_sku_item when loop reach the last size
yield Request(sku_item_url_list[0],
callback=self.parse_sku_item,
meta={
"sku_size_list": sku_size_list,
"sku_item_url_list": sku_item_url_list,
"item": item,
"index": index
})
else:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = skusDict['product']['id']
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['color'] = u'one color'
skuItem['size'] = u'one size'
skuItem['id'] = skusDict['product']['sku_code']
skuItem['from_site'] = item['from_site']
if skusDict['product']['stock'] == 0:
skuItem['is_outof_stock'] = True
item['skus'].append(skuItem)
item['sizes'] = [u'one size']
yield item
def handle_parse_item(self, response, baseItem):
sel = Selector(response)
product_id = sel.xpath('//div[@id="productId"]/text()').extract()[0]
baseItem['gender'] = 'men'
baseItem['type'] = 'base'
baseItem['from_site'] = self.name
baseItem['show_product_id'] = product_id
baseItem['title'] = sel.xpath(
'//span[@class="row product-title"]/text()').extract()[0].strip()
size_fit_container = sel.xpath('//div[@id="sizeFitContainer"]')
if len(size_fit_container) > 0:
size_fit = size_fit_container.extract()[0]
baseItem['desc'] = '<div>' + sel.xpath(
'//div[@itemprop="description"]').extract(
)[0] + size_fit + "</div>"
else:
baseItem['desc'] = sel.xpath(
'//div[@itemprop="description"]').extract()[0]
baseItem['dimensions'] = ['size', 'color']
skus = []
product_detail_str = "".join(
re.findall(r"var\s+productDetail[^;]+", response.body))
if len(product_detail_str) > 0:
context = execjs.compile('''
%s
function get_product_detail(){
return productDetail;
}
''' % (product_detail_str))
product_detail = context.call('get_product_detail')
size_js_infos = product_detail['sizes']
size_infos = {}
size_values = []
for size_id in size_js_infos:
size_infos[size_js_infos[size_id]['sizeCode']] = size_id
size_values.append(size_id)
list_price = sel.xpath(
'//div[@id="productPrices"]//meta[@itemprop="price"]/@content'
).extract()[0]
color_price_blocks = sel.xpath(
'//div[@id="productPrices"]//div[@class="priceBlock"]')
# color_price_mapping = {}
# for color_price_block in color_price_blocks:
# color_name = color_price_block.xpath(
# './span[@class="priceColors"]/text()').extract()
# if len(color_name) > 0:
# regular_price_span = color_price_block.xpath(
# './span[@class="regularPrice"]/text()').extract()
# if len(regular_price_span) > 0:
# color_price_mapping[color_name[0]] = regular_price_span[0]
# else:
# color_price_mapping[color_name[0]] = color_price_block.xpath(
# './span[@class="salePrice"]/text()').extract()[0]
match = re.search(r'productPage\.sellingPrice\=\'([\d\.]+)\';',
response.body)
if match is None:
current_price = list_price
else:
current_price = match.group(1)
image_items = product_detail['colors']
color_names = []
for key in image_items:
imageItems = image_items[key]['images']
color_name = image_items[key]['colorName'].strip() + '-' + str(key)
color_names.append(color_name)
images = []
tmp_images = []
for image_key in imageItems:
imageItem = ImageItem()
image = imageItems[image_key]
imageItem['thumbnail'] = image['thumbnail']
imageItem['image'] = image['zoom']
tmp_images.append((image['index'], imageItem))
tmp_images = sorted(tmp_images, key=lambda x: x[0])
for tmp_tuple in tmp_images:
images.append(tmp_tuple[1])
colorItem = Color()
colorItem['type'] = 'color'
colorItem['show_product_id'] = baseItem['show_product_id']
colorItem['from_site'] = self.name
colorItem['cover'] = image_items[key]['swatch']
colorItem['name'] = color_name
colorItem['images'] = images
yield colorItem
sizes = image_items[key]['sizes']
for size in sizes:
size_name = size_infos[size]
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['from_site'] = self.name
skuItem['color'] = color_name
skuItem['show_product_id'] = baseItem['show_product_id']
skuItem['id'] = key + "-" + size
skuItem['size'] = size_name
skuItem['list_price'] = list_price
skuItem['current_price'] = current_price
# if len(color_price_mapping) > 0 and color_name in color_price_mapping.keys():
# skuItem['current_price'] = color_price_mapping[
# colorItem['name']]
# else:
# skuItem['current_price'] = skuItem['list_price']
skuItem['is_outof_stock'] = False
skus.append(skuItem)
baseItem['sizes'] = size_values
baseItem['colors'] = color_names
baseItem['skus'] = skus
product_items = sel.xpath(
'//ul[@id="similarities"]/li[@class="product"]')
if len(product_items) > 0:
related_items_id = []
for product_item in product_items:
product_id = product_item.xpath(
'./div/div[@class="info"]/img/@data-product-id').extract(
)[0]
related_items_id.append(product_id)
if related_items_id:
baseItem['related_items_id'] = related_items_id
yield baseItem
def handle_parse_item(self, response, baseItem):
product_detail_str="".join(re.findall(r"var\s+productDetail[^;]+", response.body))
if len(product_detail_str)>0:
context = execjs.compile('''
%s
function get_product_detail(){
return productDetail;
}
''' % (product_detail_str))
product_detail = context.call('get_product_detail')
sel = Selector(response)
product_id = sel.xpath('//div[@id="productId"]/text()').extract()[0]
skus = []
baseItem['from_site'] = self.name
baseItem['show_product_id'] = product_id
size_js_infos = product_detail['sizes']
size_infos = {}
size_values = []
for size_id in size_js_infos:
size_infos[size_js_infos[size_id]['sizeCode']] = size_id
size_values.append(size_id)
list_price = sel.xpath('//div[@id="productPrices"]//meta[@itemprop="price"]/@content').extract()[0]
color_price_blocks = sel.xpath('//div[@id="productPrices"]//div[@class="priceBlock"]')
color_price_mapping = {}
for color_price_block in color_price_blocks:
color_name = color_price_block.xpath('./span[@class="priceColors"]/text()').extract()
if len(color_name) > 0:
regular_price_span = color_price_block.xpath('./span[@class="regularPrice"]/text()').extract()
if len(regular_price_span) > 0:
color_price_mapping[color_name[0]] = regular_price_span[0]
else:
color_price_mapping[color_name[0]] = color_price_block.xpath('./span[@class="salePrice"]/text()').extract()[0]
image_items = product_detail['colors']
color_names = []
for key in image_items:
imageItems = image_items[key]['images']
color_name = image_items[key]['colorName'].strip()
color_names.append(color_name)
images=[]
tmp_images = []
for image_key in imageItems:
imageItem = ImageItem()
image = imageItems[image_key]
imageItem['thumbnail'] = image['thumbnail']
imageItem['image'] = image['zoom']
tmp_images.append((image['index'], imageItem))
tmp_images = sorted(tmp_images, key=lambda x:x[0])
for tmp_tuple in tmp_images:
images.append(tmp_tuple[1])
colorItem = Color()
colorItem['type'] = 'color'
colorItem['show_product_id'] = product_id
colorItem['from_site'] = self.name
colorItem['cover'] = image_items[key]['swatch']
colorItem['name'] = color_name
colorItem['images'] = images
yield colorItem
sizes = image_items[key]['sizes']
for size in sizes:
size_name = size_infos[size]
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['from_site'] = self.name
skuItem['color'] = color_name
skuItem['show_product_id'] = product_id
skuItem['id'] = key+"-"+size
skuItem['size'] = size_name
skuItem['list_price'] = list_price
if len(color_price_mapping)>0 and color_name in color_price_mapping.keys():
# skuItem['current_price'] = sale_price_span.re(r'\d+.?\d*')[0]
skuItem['current_price'] = color_price_mapping[colorItem['name']]
else:
skuItem['current_price'] = skuItem['list_price']
skuItem['is_outof_stock'] = False
skus.append(skuItem)
baseItem['sizes'] = size_values
baseItem['colors']= color_names
baseItem['skus'] = skus
size_fit_container = sel.xpath('//div[@id="sizeFitContainer"]')
if len(size_fit_container)>0:
size_fit = size_fit_container.extract()[0]
baseItem['desc'] = '<div>'+sel.xpath('//div[@itemprop="description"]').extract()[0]+size_fit+"</div>"
else:
baseItem['desc'] = sel.xpath('//div[@itemprop="description"]').extract()[0]
baseItem['dimensions'] = ['size', 'color']
yield baseItem
def handle_parse_item(self, response, item):
sel = Selector(response)
if len(sel.xpath('//h1[@class="pageTitle"]/text()')) > 0:
return
if len(sel.xpath('//section[@class="productDesc"]')) > 0:
return
"""单品s"""
if len(
sel.xpath(
'//div[@class="productQuickDetails"]//p[@class="productPrice sale textSale"]/span/text()'
)) > 0:
item['current_price'] = sel.xpath(
'//div[@class="productQuickDetails"]//p[@class="productPrice sale textSale"]/span/text()'
).extract()[0]
item['list_price'] = sel.xpath(
'//div[@class="productQuickDetails"]//p[@class="productPrice regular"]/span/text()'
).extract()[0]
else:
item['current_price'] = sel.xpath(
'//div[@class="productQuickDetails"]//p[@class="productPrice "]/span/text()'
).extract()[0]
item['list_price'] = sel.xpath(
'//div[@class="productQuickDetails"]//p[@class="productPrice "]/span/text()'
).extract()[0]
if '-' in item['list_price']:
item['list_price'] = item['list_price'].split('-')[-1].strip()
if '-' in item['current_price']:
item['current_price'] = item['current_price'].split(
'-')[-1].strip()
item['show_product_id'] = sel.xpath(
'//div[@class="productNumber"]/text()').extract()[0]
if 'Item' in item['show_product_id']:
item['show_product_id'] = re.search(
'(\d+)', item['show_product_id']).group(1)
item['desc'] = sel.xpath(
".//div[@class='productShortDescription']/p/text()").extract()
if len(sel.xpath('//div[@class="longDescription"]')) > 0:
item['desc'] = item['desc'] + sel.xpath(
'//div[@class="longDescription"]').extract()
if len(sel.xpath('div[@class="productSpecs"]')) > 0:
item['desc'] = item['desc'] + sel.xpath(
'div[@class="productSpecs"]').extract()
item['desc'] = ''.join(item['desc'])
variants_json_str = re.search('var variantsJson = (.+);',
response.body).group(1)
variants_json = json.loads(variants_json_str)
item['skus'] = []
item['sizes'] = []
if not variants_json['items']:
item['sizes'] = ['One Size']
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = re.search('productId: \"(\d+)\"',
response.body).group(1)
skuItem['color'] = 'One Color'
skuItem['size'] = 'One Size'
skuItem['from_site'] = self.name
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
else:
for variant_json in variants_json['items']:
item['sizes'].append(variant_json['id'])
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = variant_json['catEntryId']
skuItem['color'] = 'One Color'
skuItem['size'] = variant_json['id']
skuItem['from_site'] = self.name
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = variant_json['price']
skuItem['is_outof_stock'] = not variant_json['buyable']
item['skus'].append(skuItem)
if 'sizeChart' in variants_json['variants'][0].keys():
item['size_chart'] = 'https:' + variants_json['variants'][0][
'sizeChart']
img_as = sel.xpath(".//a[@class='productThumb thumbnailImage']")
images = []
if len(img_as) > 0:
for img_a in img_as:
imageItem = ImageItem()
imageItem['image'] = img_a.xpath("./img/@src").extract()[0]
imageItem['thumbnail'] = img_a.xpath("./img/@src").extract()[0]
if '?' in imageItem['image']:
imageItem['image'] = imageItem['image'].split(
'?')[0] + self.image_url_suffix
if '?' in imageItem['thumbnail']:
imageItem['thumbnail'] = imageItem['thumbnail'].split(
'?')[0] + self.cover_url_suffix
images.append(imageItem)
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = self.name
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = 'One Color'
colorItem['cover'] = images[0]['thumbnail']
colorItem['images'] = images
yield colorItem
item['colors'] = ['One Color']
if 'size_chart' in item.keys():
yield Request(item['size_chart'],
callback=self.parse_size_chart,
meta={"item": item},
dont_filter=True)
else:
yield item
def handle_parse_item(self, response, item):
skus = []
sel = Selector(response)
item['from_site'] = self.name
if 'whoops' in response.url:
logging.warning('anti scraping: ' + response.url)
match = re.search(r'"product_id":\s*\[\s*"([^"]+)"\s*\]',
response.body)
if match is None:
return
temp_show_product_id = match.group(1)
current_price = sel.xpath(
'//span[contains(@class, "price-sales")]/text()').extract()
if len(current_price) > 0:
current_price = current_price[0]
list_price = sel.xpath(
'//span[contains(@class, "price-standard")]/text()').re(
r'(\S+)')
if len(list_price) > 0:
list_price = list_price[0]
else:
list_price = current_price
item['brand'] = self.name
# item['desc']=".".join(sel.xpath('//div[contains(@class, "additional")]/ul/li/text()').extract())
desc1 = sel.xpath(
'//div[@class="categorylisting detail"]/div/div').extract()[0]
desc2 = sel.xpath(
'//div[@class="categorylisting fabric"]/div/div').extract()[0]
item['desc'] = re.sub(r'[\t\n]', '', desc1 + desc2)
item['desc'] = re.sub('<img.+?>', '', item['desc'])
if sel.xpath(
'//div[contains(@class, "quantity clearfix")]//p[contains(@class, "in-stock-msg")]/text()'
):
colors = []
item_colors_links = sel.xpath(
'//div[@id="product-content"]//ul[contains(@class, "swatches color")]//li[contains(@class,"selected")]/a'
)
item_sizes = sel.xpath(
'//div[@id="product-content"]//div[contains(@class, "value")]//ul[contains(@class, "swatches size")]/li[@class!="emptyswatch unselectable"]//@title'
).extract()
item['sizes'] = item_sizes
item['dimensions'] = ['size']
item['product_type'] = 'mother-baby'
if len(item_colors_links) == 0:
item_colors_links = ['one_color']
for item_color_link in item_colors_links:
images = []
thumbnails = sel.xpath(
'//div[@id="thumbnails"]//li[@class!="thumb pdpvideo"]'
)
if thumbnails:
for li in thumbnails:
#thumbnails_evl=li.xpath('./a/img/@src').extract()[0]
imageItem = ImageItem()
image_url = li.xpath('./a/img/@src').extract()[0]
imageItem['image'] = self.handle_image_url(
image_url.encode('utf-8'), 1000, 1000)
imageItem['thumbnail'] = self.handle_image_url(
image_url.encode('utf-8'), 350, 350)
images.append(imageItem)
elif sel.xpath(
'//div[@id="thumbnails"]/li[@class="thumb pdpvideo"]/a/img/@src'
):
imageItem = ImageItem()
image_url = sel.xpath(
'//img[@class="primary-image"]/@src').extract()[0]
imageItem['image'] = self.handle_image_url(
image_url.encode('utf-8'), 1000, 1000)
imageItem['thumbnail'] = self.handle_image_url(
image_url.encode('utf-8'), 350, 350)
images.append(imageItem)
else:
imageItem = ImageItem()
image_url = sel.xpath(
'//img[@class="primary-image"]/@src').extract()[0]
imageItem['image'] = self.handle_image_url(
image_url.encode('utf-8'), 1000, 1000)
imageItem['thumbnail'] = self.handle_image_url(
image_url.encode('utf-8'), 350, 350)
images.append(imageItem)
if len(item_colors_links) > 0:
color_name = item_color_link.xpath(
'./@title').extract()[0]
color_cover = item_color_link.xpath('./@style').re(
'http://[^\)]+')[0]
else:
color_name = 'one_color'
color_cover = images[0]['thumbnail']
colors.append(color_name)
show_product_id = temp_show_product_id + "*" + color_name
item['show_product_id'] = show_product_id
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = show_product_id
color['images'] = images
color['name'] = color_name
color['cover'] = color_cover
yield color
for item_size in item_sizes:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = show_product_id
skuItem[
'id'] = item['show_product_id'] + "*" + item_size
skuItem['current_price'] = current_price
skuItem['list_price'] = list_price
if len(item_colors_links) > 0:
skuItem['color'] = item_color_link.xpath(
'./@title').extract()[0]
else:
skuItem['color'] = 'one_color'
skuItem['size'] = item_size
skuItem['from_site'] = self.name
skuItem['is_outof_stock'] = False
skuItem['quantity'] = sel.xpath(
'//select[contains(@id, "Quantity")]//@value'
).extract()[0]
#yield skuItem
skus.append(skuItem)
item['colors'] = colors
item['skus'] = skus
yield item
def handle_parse_item(self, response, item):
pImgStr = "".join(re.findall(r'(pImgs[^;]+;)+', response.body))
context = execjs.compile('''
%s
function getPImgs(){
return pImgs;
}
''' % pImgStr)
pImgs = context.call('getPImgs')
sel = Selector(response)
outofstock_result = re.search(r'outOfStock[\s]*=[\s]*([^;]+);',
response.body)
if outofstock_result and outofstock_result.group(1) == 'true':
return
stock_json_result = re.search(r'var stockJSON[\s]*=[\s]*([^;]+);',
response.body)
if stock_json_result:
stock_dic = eval(stock_json_result.group(1))
if stock_dic:
color_price_dic = eval(
re.search(r'colorPrices[\s]*=[\s]*([^;]+);',
response.body).group(1))
style_id_dic = eval(
re.search(r'styleIds[\s]*=[\s]*([^;]+);',
response.body).group(1))
product_gender = eval(
re.search(r'productGender[\s]*=[\s]*([^;]+);',
response.body).group(1))
zeta_categories = eval(
re.search(r'zetaCategories[\s]*=[\s]*([^;]+);',
response.body).group(1))
category = eval(
re.search(r';[\s]*category[\s]*=[\s]*([^;]+);',
response.body).group(1))
sub_category = eval(
re.search(r'subCategory[\s]*=[\s]*("[^"]+"[\s]*);',
response.body).group(1))
dimension_dic = eval(
re.search(r'dimensions[\s]*=[\s]*([^;]+);',
response.body).group(1))
#dimToUnitToValJSON = eval(re.search(r'dimToUnitToValJSON[\s]*=[\s]*([^;]+);', response.body).group(1))
dimensionIdToNameJson = eval(
re.search(r'dimensionIdToNameJson[\s]*=[\s]*([^;]+);',
response.body).group(1))
valueIdToNameJSON = eval(
re.search(r'valueIdToNameJSON[\s]*=[\s]*([^;]+);',
response.body).group(1))
colorNames = eval(
re.search(r'colorNames[\s]*=[\s]*({[^}]+}[\s]*);',
response.body).group(1))
if len(zeta_categories) > 0:
item['product_type'] = zeta_categories[0].values()[0]
if category == item['product_type']:
item['category'] = sub_category
else:
item['category'] = category
item['sub_category'] = sub_category
else:
item['product_type'] = category
item['category'] = sub_category
if 'gender' in response.meta.keys():
meta_gender = response.meta['gender']
if product_gender.lower == 'unisex':
if meta_gender == 'boys' or meta_gender == 'girls':
item['gender'] = 'kid-unisex'
else:
item['gender'] = 'unisex'
else:
item['gender'] = meta_gender
'''跳过描述，过于复杂'''
size_info_images = []
desc = sel.xpath(
'//div[@id="productDescription"]//div[@itemprop="description"]/ul'
).extract()
if len(desc) > 0:
item['desc'] = desc[0]
size_infos = sel.xpath(
'//div[@id="productDescription"]//div[@itemprop="description"]/ul/li/a[@class="popup-570-550"]'
)
if len(size_infos) > 0:
size_info_images = []
for size_info in size_infos:
size_info_image_url = size_info.xpath(
'@href').extract()[0]
if not re.match(r'^http:\/\/', size_info_image_url):
size_info_image_url = self.base_url + size_info_image_url
size_info_images.append(size_info_image_url)
else:
desc_ul = sel.xpath(
'//div[@id="prdInfoText"]//span[@class="description summary"]/ul'
).extract()
if len(desc_ul) == 0:
return
item['desc'] = desc_ul[0]
size_infos = sel.xpath(
'//div[@id="prdInfoText"]//span[@class="description summary"]/ul/li/a[@class="popup-570-550"]'
)
if len(size_infos) > 0:
size_info_images = []
for size_info in size_infos:
size_info_image_url = size_info.xpath(
'@href').extract()[0]
if not re.match(r'^http:\/\/', size_info_image_url):
size_info_image_url = self.base_url + size_info_image_url
size_info_images.append(size_info_image_url)
if len(size_info_images) > 0:
item['size_info'] = {'images': size_info_images}
colors = []
'''处理color'''
for (color, color_name) in colorNames.items():
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = self.name
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = color_name
colors.append(color_name)
styleId = str(style_id_dic[color])
#colorItem['cover'] = sel.xpath('//a[@id="frontrow-'+color+'"]/img/@src').extract()[0]
if 'p' in pImgs[styleId]['DETAILED'].keys():
colorItem['cover'] = pImgs[styleId]['DETAILED']['p']
elif 'd' in pImgs[styleId]['DETAILED'].keys():
colorItem['cover'] = pImgs[styleId]['DETAILED']['d']
elif '1' in pImgs[styleId]['MULTIVIEW_THUMBNAILS'].keys():
colorItem['cover'] = pImgs[styleId][
'MULTIVIEW_THUMBNAILS']['1']
elif '4' in pImgs[styleId]['MULTIVIEW_THUMBNAILS'].keys():
colorItem['cover'] = pImgs[styleId][
'MULTIVIEW_THUMBNAILS']['4']
elif '5' in pImgs[styleId]['MULTIVIEW_THUMBNAILS'].keys():
colorItem['cover'] = pImgs[styleId][
'MULTIVIEW_THUMBNAILS']['5']
colorImages = pImgs[styleId]
thumbImages = colorImages['MULTIVIEW_THUMBNAILS']
images = colorImages['2x']
if len(images) == 0:
images = colorImages['MULTIVIEW']
thumbImages = sorted(thumbImages.iteritems(),
key=lambda d: d[0])
images_array = []
for image_tuple in thumbImages:
imageItem = ImageItem()
if image_tuple[0] in images.keys():
imageItem['image'] = images[image_tuple[0]]
imageItem['thumbnail'] = image_tuple[1]
if image_tuple[0] == 'p' or image_tuple[0] == 'd':
images_array.insert(0, imageItem)
else:
images_array.append(imageItem)
colorItem['images'] = images_array
yield colorItem
item['colors'] = colors
dimensions = []
sizes = {}
for dimension in dimension_dic:
dimensions.append(dimensionIdToNameJson[dimension])
sizes[dimensionIdToNameJson[dimension]] = []
if len(dimensions) == 0:
dimensions = ['size']
if len(sizes) == 0:
sizes = {'size': ['onesize']}
item['dimensions'] = dimensions
'''处理sku库存'''
skuCollectionsList = []
for sku_stock in stock_dic:
color = sku_stock['color']
if color in color_price_dic.keys():
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['from_site'] = self.name
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = sku_stock['id']
skuItem["list_price"] = color_price_dic[color]['wasInt']
skuItem['current_price'] = color_price_dic[color]['nowInt']
skuItem['color'] = colorNames[color]
size_demension = {}
for demension in dimension_dic:
if demension in sku_stock.keys(
) and sku_stock[demension] in valueIdToNameJSON.keys():
size_value = valueIdToNameJSON[
sku_stock[demension]]['value']
size_demension[
dimensionIdToNameJson[demension]] = size_value
if not size_value in sizes[
dimensionIdToNameJson[demension]]:
sizes[dimensionIdToNameJson[demension]].append(
size_value)
if len(size_demension) == 0:
size_demension = {'size': 'onesize'}
skuItem['size'] = size_demension
skuItem['quantity'] = sku_stock['onHand']
skuItem['is_outof_stock'] = False
skuCollectionsList.append(skuItem)
item['skus'] = skuCollectionsList
item['sizes'] = sizes
item = self.handle_dimension_to_name(response, item,
dimensionIdToNameJson)
yield item
def handle_parse_item(self, response, item):
body_json = json.loads(response.body)
goods_detail = body_json['data']
if goods_detail['inStock'] == 0:
return
item['linkhaitao_url'] = response.url
item['cover'] = goods_detail['coverImgUrl']
item['desc'] = goods_detail['content']['description']
if 'product_type_id' in os.environ.keys():
self.product_type_id = os.environ['product_type_id']
if 'category_id' in os.environ.keys():
self.category_id = os.environ['category_id']
if self.product_type_id:
item['product_type_id'] = int(self.product_type_id)
item['product_type'] = 'linkhaitao_' + str(self.product_type_id)
if self.category_id:
item['category_id'] = int(self.category_id)
item['category'] = 'linkhaitao_' + str(self.category_id)
if 'editor_flag' in os.environ.keys():
self.editor_flag = os.environ['editor_flag']
if self.editor_flag:
item['editor_flag'] = self.editor_flag
if 'gender' in os.environ.keys():
self.gender = os.environ['gender']
if self.gender:
item['gender'] = self.gender
item['dimensions'] = ['size', 'color']
item['brand'] = goods_detail['brand']['name_en']
item['title'] = goods_detail['name']
item['current_price'] = goods_detail['realPriceOrg']
item['list_price'] = goods_detail['mallPriceOrg']
from_site = ''.join(goods_detail['sellerName']['namecn'].split()).lower()
if self.is_chinese_word(from_site):
from_site = ''.join(goods_detail['sellerName']['namecn'].split()).lower()
if "'" in from_site:
from_site = from_site.replace("'", "")
if '/' in from_site:
from_site = from_site.split('/')[0]
item['from_site'] = from_site
if item['from_site'] == '6pm' or item['from_site'] == '6pm/6pm':
item['from_site'] = 'sixpm'
spu_id = re.search('spuid=(.+)&?',response.url)
if spu_id:
spu_id = spu_id.group(1)
else:
spu_id = re.search('&spu=(.+)&?',response.url).group(1)
item['show_product_id'] = spu_id
item['url'] = goods_detail['pageUrl']
if self.editor_flag:
item['editor_flag'] = self.editor_flag
if not goods_detail['skuInfo']:
colorItem = Color()
images = []
color_names = []
skus=[]
for image in goods_detail['coverImgList']:
imageItem = ImageItem()
imageItem['image'] = image
imageItem['thumbnail'] = image
images.append(imageItem.copy())
colorItem['images'] = images
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
color_name = 'One Color'
if color_name not in color_names:
color_names.append(color_name)
colorItem['name'] = color_name
colorItem['cover'] = goods_detail['coverImgUrl']
yield colorItem
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['color'] = color_name
skuItem['id'] = item['show_product_id'] + 'onecolor'
skuItem['from_site'] = item['from_site']
if goods_detail['inStock'] == 0:
skuItem['is_outof_stock'] = True
skuItem['size'] = 'One Size'
skus.append(skuItem)
item['sizes'] = ['One Size']
else:
skus_info = goods_detail['skuInfo']['style']['skustylelist']
color_names = []
skus = []
sizes = []
dimensions_values = {}
for sku_info in skus_info:
colorItem = Color()
images = []
for image in sku_info['coverImgList']:
imageItem = ImageItem()
imageItem['image'] = image
imageItem['thumbnail'] = image
images.append(imageItem.copy())
colorItem['images'] = images
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
if sku_info['style']:
color_name = sku_info['style']
else:
color_name = 'One Color'
if color_name not in color_names:
color_names.append(color_name)
colorItem['name'] = color_name
colorItem['cover'] = images[0]['image']
yield colorItem
for sku in sku_info['data']:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = sku['mallPriceOrg']
skuItem['current_price'] =sku['realPriceOrg']
skuItem['color'] = color_name
skuItem['id'] = sku['skuid']
skuItem['from_site'] = item['from_site']
if sku['inStock'] == 0:
skuItem['is_outof_stock'] = True
if len(sku['attr']) == 0:
skuItem['size'] = 'One Size'
if skuItem['size'] not in sizes:
sizes.append(skuItem['size'])
else:
skuItem['size'] = {}
for attr in sku['attr']:
skuItem['size'][attr['attrName'].lower()] = attr['attrValue']
if attr['attrName'].lower() not in item['dimensions']:
item['dimensions'].append(attr['attrName'].lower())
if attr['attrName'].lower() not in dimensions_values.keys():
dimensions_values[attr['attrName'].lower()] = [attr['attrValue']]
else:
dimensions_values[attr['attrName'].lower()].append(attr['attrValue'])
if 'size' not in skuItem['size'].keys():
skuItem['size']['size'] = 'One Size'
dimensions_values['size'] = 'One Size'
skus.append(skuItem)
if sizes:
item['sizes'] = sizes
elif dimensions_values:
item['sizes'] = dimensions_values
else:
return
item['skus'] = skus
item['colors'] = color_names
yield item
