def parse_colors(self, colorsDom, itemB):
color_item_url_mapping = {}
color_names = []
for dom in colorsDom:
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = itemB['from_site']
colorItem['show_product_id'] = itemB['show_product_id']
colorItem['name'] = dom.xpath("./a/@title").extract()[0]
color_names.append(colorItem['name'])
style = re.findall(r'url\((http://.*)\)',
dom.xpath("./a/@style").extract()[0])
if len(style) > 0:
colorItem['cover'] = style[0]
else:
colorItem['cover'] = ''
colorUrl = dom.xpath("./a/@data-href").extract()[0]
color_item_url_mapping[colorUrl] = colorItem
itemB['colors'] = color_names
index = 0
# import pdb;pdb.set_trace()
url = color_item_url_mapping.keys()[index]
yield Request(url,
callback=self.parse_item_B,
meta={
"item": itemB,
"color_item_url_mapping": color_item_url_mapping,
'index': index
})
def parse_color_item(self, response):
sel = Selector(response)
index = response.meta['index']
color_urls = response.meta['color_urls']
baseItem = response.meta['baseItem']
colorItem = Color()
colorItem['type'] = 'color'
colorItem['show_product_id'] = baseItem['show_product_id']
colorItem['from_site'] = self.name
colorItem['name'] = color_urls[index]['color_name']
colorItem['cover'] = color_urls[index]['color_cover']
images = []
imageItem = ImageItem()
image_url = sel.xpath('//meta[@property="og:image"]/@content').extract()[0]
imageItem['image'] = re.sub(r'wid=\d+&hei=\d+', 'wid=1000&hei=1000', image_url)
imageItem['thumbnail'] = re.sub(r'wid=\d+&hei=\d+', 'wid=50&hei=50', image_url)
images.append(imageItem)
image_url2 = sel.xpath('//div[@id="productSwatch"]/img/@src').extract()[0]
imageItem = ImageItem()
imageItem['image'] = re.sub(r'wid=\d+&hei=\d+', 'wid=1000&hei=1000', image_url2)
imageItem['thumbnail'] = re.sub(r'wid=\d+&hei=\d+', 'wid=50&hei=50', image_url2)
images.append(imageItem)
colorItem['images'] = images
yield colorItem
skus = response.meta['skus']
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = baseItem['show_product_id']
skuItem['from_site'] = self.name
skuItem['current_price'] = sel.xpath('//div[@class="prodprice saleprice"]/p/span[@itemprop="price"]/text()').extract()[0]
if len(sel.xpath('//div[@class="prodprice saleprice"]/p/span[@class="basePrice"]/text()').extract()) > 0:
skuItem['list_price'] = sel.xpath('//div[@class="prodprice saleprice"]/p/span[@class="basePrice"]/text()').extract()[0]
else:
skuItem['list_price'] = skuItem['current_price']
skuItem['is_outof_stock'] = False
skuItem['color'] = color_urls[index]['color_name']
skuItem['size'] = 'one-size'
skuItem['id'] = baseItem['show_product_id']
skus.append(skuItem)
if index + 1 == len(color_urls):
baseItem['skus'] = skus
yield baseItem
else:
yield Request(color_urls[index+1]['url'], callback=self.parse_color_item
, meta={'baseItem': baseItem, 'color_urls': color_urls, 'index': index+1, 'skus': skus})
def parse_color_sku(self, response):
baseItem = response.meta['baseItem']
images_tmp = response.meta['images']
jsonStr = json.loads(response.body)
colors = []
skus = []
sizes = []
for col in jsonStr['Colors']:
images = []
for img in images_tmp:
imageItem = ImageItem()
imageItem['thumbnail'] = '%s%s_%s_%s.%s' % (
img['base'], col['Code10'], img['thum_size'], img['index'],
img['thum_ext'])
imageItem['image'] = '%s%s_%s_%s.%s' % (
img['base'], col['Code10'], img['img_size'], img['index'],
img['img_ext'])
images.append(imageItem)
color = Color()
color['type'] = 'color'
color['from_site'] = 'thecorner'
color['show_product_id'] = baseItem['show_product_id']
color['images'] = images
color['name'] = col['Description']
color['cover_style'] = '#' + col['Rgb']
#color['cover_style'] = 'background-color: #%s;' % (col['Rgb'])
colors.append(col['Description'])
yield color
for size in jsonStr['ModelColorSizes']:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = baseItem['show_product_id']
skuItem['from_site'] = 'thecorner'
skuItem['id'] = size['Color']['Description'].encode(
"utf-8") + "*" + size['Size']['Description']
skuItem['list_price'] = baseItem['list_price']
skuItem['current_price'] = baseItem['current_price']
skuItem['size'] = size['Size']['Description']
skuItem['color'] = size['Color']['Description']
skuItem['is_outof_stock'] = False
skuItem['quantity'] = size['Quantity']
sizes.append(size['Size']['Description'])
skus.append(skuItem)
baseItem['skus'] = skus
baseItem['colors'] = list(set(colors))
baseItem['sizes'] = list(set(sizes))
yield baseItem
def parse_color(self, response):
item = response.meta['item']
sel = Selector(response)
color_images = sel.xpath(
'//div[@id="largerViewContainer"]/table[3]/tr[2]/td/a')
images = []
cover = ''
if len(color_images) > 0:
for color_image in color_images:
image_id = color_image.xpath('@onmouseover').re(
r'AltView\(\'([\d]+)\'\,')[0]
thumbnail = self.base_url.strip('/') + color_image.xpath(
'img/@src').extract()[0]
if image_id == '0':
image = self.base_url + 'products/' + item[
'show_product_id'] + '_xl.jpg'
cover = thumbnail
else:
image = self.base_url + 'products/alt_views/' + item[
'show_product_id'] + '_alt' + image_id + '.jpg'
images.append({'image': image, 'thumbnail': thumbnail})
else:
images = [{
'image':
self.base_url + 'products/' + item['show_product_id'] +
'_xl.jpg',
'thumbnail':
self.base_url + 'products/' + item['show_product_id'] +
'_xs.jpg'
}]
cover = self.base_url + 'products/' + item[
'show_product_id'] + '_xs.jpg'
image_item = Color()
image_item['show_product_id'] = item['show_product_id']
image_item['from_site'] = item['from_site']
image_item['type'] = 'color'
image_item['cover'] = cover
image_item['name'] = 'onecolor'
image_item['images'] = images
yield image_item
def handle_parse_item(self, response, baseItem):
sel = Selector(response)
baseItem['dimensions'] = ['size', 'color']
baseItem['desc'] = sel.xpath(
'//div[@id="aDescriptionBody"]').extract()[0]
baseItem['brand'] = sel.xpath(
'//span[@itemprop="brand"]/text()').extract()[0]
skus = []
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = baseItem['show_product_id']
skuItem['from_site'] = self.name
skuItem['current_price'] = baseItem['current_price']
skuItem['list_price'] = baseItem['list_price']
skuItem['is_outof_stock'] = False
skuItem['color'] = 'one-color'
skuItem['size'] = 'one-size'
skuItem['id'] = baseItem['show_product_id']
skus.append(skuItem)
imageItem = ImageItem()
imageItem['image'] = 'http:' + sel.xpath(
'//img[@id="ImageUrl"]/@src').extract()[0]
imageItem['thumbnail'] = imageItem['image']
images = []
images.append(imageItem)
colorItem = Color()
colorItem['type'] = 'color'
colorItem['show_product_id'] = baseItem['show_product_id']
colorItem['from_site'] = self.name
colorItem['images'] = images
colorItem['name'] = 'one-color'
yield colorItem
baseItem['skus'] = skus
yield baseItem
def handle_parse_item(self, response, item):
current_url = get_base_url(response)
if current_url == 'http://www.levi.com/US/en_US/error' or current_url == 'http://global.levi.com':
return
buyJsonStr = "".join(
re.findall(
r'(var buyStackJSON = .*;[\s\S]*productCodeMaster\s*=[\s\S]*?;)[\s\S]*?</script>',
response.body))
context = execjs.compile('''
%s
function getBuyStackJSON(){
return buyStackJSON;
}
function getProductCodeMaster(){
return productCodeMaster;
}
''' % buyJsonStr)
stackJson = json.loads(context.call('getBuyStackJSON'))
productMaster = str(context.call('getProductCodeMaster'))
if productMaster not in stackJson['colorids']:
return
item["show_product_id"] = productMaster
if 'name' in stackJson["colorid"][productMaster].keys():
item['desc'] = stackJson["colorid"][productMaster]['name']
item['list_price'] = stackJson["colorid"][productMaster]['price'][0][
'amount']
if len(stackJson["colorid"][productMaster]['price']) > 1:
item['current_price'] = stackJson["colorid"][productMaster][
'price'][1]['amount']
else:
item['current_price'] = item['list_price']
altImgs = stackJson["colorid"][productMaster]['altViews']
altZoomImgs = stackJson["colorid"][productMaster]['altViewsZoomPaths']
imgDir = stackJson["colorid"][productMaster]['imageURL']
images = []
if len(altImgs):
imgKey = 0
for imgVal in altImgs:
if re.search('alt\d*\-pdp\.jpg', imgVal) > -1:
imgDir = re.search(r'(.+' + productMaster[:5] + r').*',
imgDir).group(1)
imgDir = imgDir + '-'
imageItem = ImageItem()
if imgVal[:6] == 'detail':
imageItem['image'] = imgDir + imgVal + stackJson[
"product"]['imageSizeDetail']
imageItem['thumbnail'] = imgDir + altZoomImgs[
imgKey] + stackJson["product"][
'imageSizeDetailThumbnail']
else:
imageItem['image'] = imgDir + imgVal + stackJson[
"product"]['imageSizeHero']
imageItem['thumbnail'] = imgDir + altZoomImgs[
imgKey] + stackJson["product"]['imageSizeThumbnail']
images.append(imageItem)
imgKey += 1
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = productMaster
colorItem['images'] = images
colorItem['name'] = stackJson["colorid"][productMaster]['finish'][
'title']
colorItem['cover'] = stackJson["colorid"][productMaster]['swatch']
color_name = colorItem['name']
yield colorItem
item['skus'] = []
item['dimensions'] = []
item['sizes'] = {}
if len(stackJson["attrs"]['waist']) > 0:
item['dimensions'].append('waist')
item['sizes']['waist'] = stackJson["attrs"]['waist']
if len(stackJson["attrs"]['length']) > 0:
item['dimensions'].append('length')
item['sizes']['length'] = stackJson["attrs"]['length']
if len(stackJson["attrs"]['size']) > 0:
item['dimensions'].append('size')
item['sizes']['size'] = stackJson["attrs"]['size']
# #有货的未处理
if stackJson["sku"]:
for sku in stackJson["sku"]:
if stackJson["sku"][sku]['colorid'] == productMaster:
is_continue = False
sku_keys = stackJson["sku"][sku].keys()
for one_dimension in item['dimensions']:
if one_dimension not in sku_keys:
is_continue = True
break
if is_continue:
continue
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = stackJson["sku"][sku][
'colorid']
skuItem['id'] = stackJson["sku"][sku]['skuid']
skuItem['list_price'] = stackJson["sku"][sku]['price'][0][
'amount']
if stackJson["sku"][sku]['finalsale']:
skuItem['current_price'] = stackJson["sku"][sku][
'price'][1]['amount']
else:
skuItem['current_price'] = skuItem['list_price']
skuItem['color'] = color_name #item['colors'][0]
if 'size' in stackJson["sku"][sku]:
skuItem['size'] = stackJson["sku"][sku]['size']
elif 'waist' in stackJson["sku"][
sku] and 'length' in stackJson["sku"][sku]:
skuItem['size'] = {
'waist': stackJson["sku"][sku]['waist'],
'length': stackJson["sku"][sku]['length']
}
elif 'sizedescription' in stackJson["sku"][sku]:
skuItem['size'] = stackJson["sku"][sku][
'sizedescription']
skuItem['from_site'] = 'levi'
skuItem['quantity'] = stackJson["sku"][sku]['stock']
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
yield skuItem
# print skuItem
# print '* ' * 20
item['colors'] = [color_name]
sel = Selector(response)
item['brand'] = sel.xpath(
".//div[@id='main-container']//meta[@itemprop='brand']/@content"
).extract()[0]
size_chart_dom = sel.xpath(
".//div[@class='pdp-sizing']/div[@class='pdp-sizes pdp-size-sizes']/a"
)
if len(size_chart_dom.extract()):
size_chart = size_chart_dom.xpath("./p/text()").extract()[0]
if size_chart == 'Size Chart':
item['size_info'] = size_chart_dom.xpath(
"./@href").extract()[0]
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
catalogId = sel.xpath(
'//input[contains(@id, "catalogId")]/@value').extract()[0]
storeId = sel.xpath(
'//input[contains(@id, "storeId")]/@value').extract()[0]
show_product_id = sel.xpath(
'//input[contains(@id, "productId")]/@value').extract()[0]
#baseItem = response.meta['baseItem']
baseItem = item
baseItem['from_site'] = self.name
baseItem['type'] = 'base'
baseItem['title'] = sel.xpath(
'//div[contains(@class, "pdd_title box")]//h3/text()').extract()[0]
baseItem['show_product_id'] = show_product_id
baseItem['desc'] = sel.xpath(
'//div[contains(@class, "pdd_desc pdd_sub_item box")]').extract(
)[0]
baseItem['list_price'] = sel.xpath(
'//p[contains(@class, "pdd_price")]//span/text()').extract()[0]
if sel.xpath('//span[contains(@class, "promo_price")]'):
baseItem['current_price'] = sel.xpath(
'//span[contains(@class, "promo_price")]/text()').extract()[0]
else:
baseItem['current_price'] = baseItem['list_price']
colors = []
skus = []
colorNames = []
sizes = {'size': ['onesize']}
color_coloum = sel.xpath(
'//ul[contains(@class, "pdd_colors_list box")]//li')
images_coloum = sel.xpath(
'//div[contains(@id, "rl_pdd_cover_slider")]//ul[contains(@class, "box")]//li'
)
for colors_col in color_coloum:
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = show_product_id
color['cover'] = colors_col.xpath('./a/img/@src').extract()[0]
color['name'] = colors_col.xpath('./a/img/@alt').extract()[0]
colorNames.append(color['name'])
if colors_col.xpath(
'./a[contains(@class, "pdd_color pdd_color_picked")]//span/@onclick'
):
images = []
for images_col in images_coloum:
imageItem = ImageItem()
imageItem['image'] = images_col.xpath(
'./img/@src').extract()[0]
imageItem['thumbnail'] = images_col.xpath(
'./img/@small').extract()[0]
images.append(imageItem)
color['images'] = images
yield color
else:
#print re.findall(r'\(.*?\)', clickParam)
clickParam = colors_col.xpath(
'./a[contains(@class, "pdd_color")]//span/@onclick'
).extract()[0]
clickParams = re.findall(r"'(.*)'", clickParam)[0].split(',')
MFPARTNUMBER = clickParams[5].replace("'", "")
imgUrl = '%s/webapp/wcs/stores/servlet/ProductDetailFullImageView?catalogId=%s&langId=-1&storeId=%s&MFPARTNUMBER=%s' % (
self.base_url, catalogId, storeId, MFPARTNUMBER)
yield Request(imgUrl.encode('UTF-8'),
meta={'color': color},
callback=self.parse_img)
'''
quantityUrl = 'http://www.ralphlauren.asia/webapp/wcs/stores/servlet/ProductDetailQuantityView?catalogId=12551&langId=-1&storeId=12151'
formdata = {
'SKUId': str(clickParam[2]),
'objectId':'',
'requesttype':'ajax'
}
yield FormRequest(url=sizeUrl, formdata=formdata, callback=self.parse_quantity, meta={ '': '' } )
sizeUrl = self.base_url + '/webapp/wcs/stores/servlet/ProductDetailSizeSelectView?catalogId=12551&langId=-1&storeId=12151'
formdata = {
'Id': str(clickParam[1]),
'SKUId': str(clickParam[2]),
'Color': 'Lime',
'ColorId': str(clickParam[5]),
'Size': '',
'InItemSppSplitChar':'@@',
'objectId':'',
'requesttype':'ajax'
}
yield FormRequest(url=sizeUrl, formdata=formdata, callback=self.parse_size, meta={ 'color': color } )
'''
###
sku_coloum = sel.xpath(
'//select[contains(@id, "rl_pdd_size")]//option ')
for sku_col in sku_coloum:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = show_product_id
skuItem['from_site'] = self.name
skuItem['list_price'] = baseItem['list_price']
skuItem['current_price'] = baseItem['current_price']
skuItem['is_outof_stock'] = False
skuItem['id'] = sel.xpath(
'//input[contains(@id, "selectSKUId")]/@value').extract()[0]
skuItem['color'] = sel.xpath(
'//span[contains(@class, "pdd_current_color")]/text()'
).extract()[0]
#skuItem['size'] = sku_col.xpath('./option/text()').extract()[0]
skuItem['quantity'] = sel.xpath(
'//select[contains(@id, "rl_pdd_qty")]//option/@value'
).extract()[0]
skus.append(skuItem)
baseItem['colors'] = colorNames
baseItem['sizes'] = sizes
baseItem['skus'] = skus
baseItem['dimensions'] = ""
baseItem['brand'] = 'ralph lauren'
baseItem['category'] = sel.xpath(
'//div[contains(@class, "bread bread_bar")]//a[4]/text()').extract(
)[0]
baseItem['product_type'] = sel.xpath(
'//div[contains(@class, "bread bread_bar")]//a[3]/text()').extract(
)[0]
yield baseItem
def handle_parse_item(self, response, item):
sel = Selector(response)
item['show_product_id'] = str(
re.search('productID: \"(\d+)\"', response.body).group(1)).strip()
item['brand'] = str(
re.search('productBrand: \"(.+)\"',
response.body).group(1)).strip()
item['title'] = sel.xpath(
'.//h1[@data-track="product-title"]/text()').extract()[0].strip()
item['desc'] = ''.join(
sel.xpath('//div[@itemprop="description"]/p').extract()).strip()
item['current_price'] = sel.xpath(
'//span[@class="price"]/text()').extract()[0].strip()
list_price_search = re.search('rrp: .+\&\#36\;([\d\.]+).+',
response.body)
if list_price_search:
item['list_price'] = list_price_search.group(1)
else:
item['list_price'] = item['current_price']
images = []
image_divs = sel.xpath(
'//div[@class="product-thumb-box productImageZoom__thumbnailContainer "]'
)
if not image_divs:
return
for image_div in image_divs:
imageItem = ImageItem()
imageItem['thumbnail'] = image_div.xpath('./img/@src').extract()[0]
imageItem['image'] = image_div.xpath(
'./parent::*/@href').extract()[0]
images.append(imageItem)
color_names = sel.xpath(
'//select[@id="opts-2"]/option[position()>1]/text()').extract()
if len(color_names) > 1:
return
if not color_names:
color_names = ['One Color']
item['colors'] = color_names
color = Color()
color['type'] = 'color'
color['from_site'] = item['from_site']
color['show_product_id'] = item['show_product_id']
color['images'] = images
color['name'] = color_names[0]
color['cover'] = images[0]['thumbnail']
yield color
skus = []
sizes = sel.xpath(
'//select[@id="opts-1"]/option[position()>1]/text()').extract()
if not sizes:
sizes = ['One Size']
item['sizes'] = sizes
for size in sizes:
for color_name in color_names:
skuItem = SkuItem()
skuItem['type'] = "sku"
skuItem['from_site'] = item['from_site']
skuItem['id'] = item[
'show_product_id'] + '-' + color_name + '-' + size
skuItem['show_product_id'] = item['show_product_id']
skuItem['current_price'] = item['current_price']
skuItem['list_price'] = item['list_price']
skuItem['size'] = size
skuItem['color'] = color_name
skus.append(skuItem)
item['skus'] = skus
item['dimensions'] = ['size']
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
if not sel.xpath('//h1[contains(@class, "prod-title")]'):
print get_base_url(response)
return
if sel.xpath('//font[@class="prodError"]'):
return
show_product_id = sel.xpath('//span[contains(@class, "style-num")]/text()').extract()[0]
baseItem = item
baseItem['type'] = 'base'
baseItem['url'] = get_base_url(response)
baseItem['title'] = sel.xpath('//h1[contains(@class, "prod-title")]/text()').extract()[0]
baseItem['cover'] = item['cover']
if sel.xpath('//span[contains(@itemprop, "offers")]//span[contains(@class, "sale-price")]'):
if sel.xpath('//span[contains(@itemprop, "offers")]//span[contains(@class, "sale-price")]//span[2]'):
baseItem['current_price'] = sel.xpath('//span[contains(@itemprop, "offers")]//span[contains(@class, "sale-price")]//span[2]/text()').extract()[0]
else:
baseItem['current_price'] = sel.xpath('//span[contains(@itemprop, "offers")]//span[contains(@class, "sale-price")]//span/text()').extract()[0]
elif sel.xpath('//span[contains(@class, "reg-price")]//span[contains(@itemprop, "price")]'):
#baseItem['current_price'] = sel.xpath('//span[contains(@itemprop, "price")]/text()').extract()[0]
baseItem['current_price'] = sel.xpath('//span[contains(@class, "reg-price")]//span[contains(@itemprop, "price")]/text()').extract()[0]
elif sel.xpath('//span[contains(@class, "reg-price")]//span[contains(@itemprop, "highPrice")]'):
baseItem['current_price'] = sel.xpath('//span[contains(@class, "reg-price")]//span[contains(@itemprop, "highPrice")]/text()').extract()[0]
else :
return
if sel.xpath('//span[contains(@class, "reg-price is-sale")]'):
list_pri = sel.xpath('//span[contains(@class, "reg-price is-sale")]/text()').extract()[0]
if list_pri.find('-') == -1:
baseItem['list_price'] = list_pri
else:
baseItem['list_price'] = list_pri.split("-")[1]
else:
baseItem['list_price'] = baseItem['current_price']
baseItem['show_product_id'] = show_product_id
baseItem['desc'] = sel.xpath('//div[contains(@class, "detail")]//ul').extract()[0]
baseItem['brand'] = 'Ralph Lauren'
baseItem['from_site'] = 'ralphlauren'
baseItem['product_type'] = item['product_type']
baseItem['category'] = item['category']
#baseItem['sub_category'] =
baseItem['gender'] = item['gender']
if sel.xpath('//a[contains(@id, "sizechart")]'):
baseItem['size_info'] = '%s%s' % (self.base_url, sel.xpath('//a[contains(@id, "sizechart")]/@href').extract()[0] )
else:
baseItem['size_info'] = ''
#jsStr = ",".join(re.findall(r'itemMap.*[\s]=[\s]*({[^}]+}[\s]*);', response.body))
####
jsStr2 = "".join(re.findall(r'<script>[\s]*(var isTablet.*;)[\s]*</script>[\s]*<div class="prod-utility">', response.body, re.S))
strinfo = re.compile('var isTablet.*;')
imgStr = strinfo.sub('var altImages = new Array();var Scene7Map = new Array();', jsStr2)
#print imgStr
context2 = execjs.compile('''
%s
function getImages(){
return orderedImageMap_0;
}
function getImages2(){
var imageArr = new Array()
for (i in altImages){
for (j in altImages[i]) {
altImages[i][j]["cId"] = i
}
imageArr.push(altImages[i])
}
return imageArr;
}
function getScene7Map(){
var Scene7Maps = new Array();
var cIds = new Array();
for (i in Scene7Map){
if (i.toString().indexOf("c") != -1 ){
cId = i.toString().substr(1, i.length-1)
cIds.push(cId)
}
}
for (ii in Scene7Map){
for (jj in cIds) {
s7Index = "s7" + cIds[jj]
if (ii == s7Index) {
Scene7Maps.push({ "cId":cIds[jj], "cValue":Scene7Map[ii]})
}
}
}
return Scene7Maps
}
''' % imgStr.decode('cp1252').encode('utf-8') )
getImages = context2.call('getImages')
getImages2 = context2.call('getImages2')
Scene7Map = context2.call('getScene7Map')
imgsArr = []
for imgTmp in getImages:
imgsArr_tmp = []
#replace pic
for STmp in Scene7Map:
if STmp['cId'] == imgTmp['cId']:
imgTmp['v400'] = 'http://s7d2.scene7.com/is/image/PoloGSI/%s?$flyout_main$&cropN=0.12,0,0.7993,1&iv=fLNd_3&wid=1410&hei=1770&fit=fit,1' % (STmp['cValue'])
imgsArr_tmp.append({"image":imgTmp['v400'], "thumbnail": imgTmp['x50']})
#video
if 'vid' in imgTmp.keys():
item['media_url'] = 'http://s7d2.scene7.com/is/content/PoloGSI/' + imgTmp['vid']
for imgTmp2 in getImages2:
for imgTmp22 in imgTmp2:
if imgTmp['cId'] == imgTmp22['cId']:
if imgTmp22['t940'] == '' and imgTmp22['x50'] != '':
imgTmp22['t940'] = imgTmp22['x50'].replace('_t50','_t940')
elif imgTmp22['t940'] != '' and imgTmp22['x50'] == '':
imgTmp22['x50'] = imgTmp22['t940'].replace('_t940','_t50')
imgsArr_tmp.append({"image":imgTmp22['t940'], "thumbnail": imgTmp22['x50']})
imgsArr.append( {"cId": imgTmp['cId'], "pics": imgsArr_tmp} )
color_col = sel.xpath('//ul[contains(@id, "color-swatches")]//li')
for colors in color_col:
color_id = colors.xpath('./@data-value').extract()[0]
cover = colors.xpath('.//img/@src').extract()[0]
name = colors.xpath('.//img/@title').extract()[0]
images = []
for img in imgsArr:
if img['cId'] == color_id:
images = img['pics']
color = Color()
color['type'] = 'color'
color['show_product_id'] = show_product_id
color['from_site'] = 'ralphlauren'
color['cover'] = cover
color['images'] = images
color['name'] = name
yield color
####
jsStr1 = "".join(re.findall(r'<script>[\s]*(var itemMap.*;)[\s]*</script>[\s]*<!--previousURL', response.body, re.S))
context1 = execjs.compile('''
%s
function getItemMaps(){
return itemMap;
}
''' % jsStr1.decode('cp1252').encode('utf-8'))
skus = []
sizes = []
colors = []
getItemMaps = context1.call('getItemMaps')
for ItemMaps in getItemMaps:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = show_product_id
skuItem['from_site'] = 'ralphlauren'
#skuItem['id'] = show_product_id + '-' +ItemMaps['sDesc']
skuItem['id'] = ItemMaps['sku']
skuItem['list_price'] = baseItem['list_price']
skuItem['current_price'] = ItemMaps['price']
skuItem['size'] = ItemMaps['sDesc']
skuItem['color'] = ItemMaps['cDesc']
if ItemMaps['avail'] == "OUT_OF_STOCK":
skuItem['is_outof_stock'] = True
else:
skuItem['is_outof_stock'] = False
skuItem['quantity'] = ItemMaps['quantityOnHand']
sizes.append(ItemMaps['sDesc'])
skus.append(skuItem)
colors.append(ItemMaps['cDesc'])
baseItem['skus'] = skus
baseItem['sizes'] = list(set(sizes))
baseItem['colors'] = list(set(colors))
yield baseItem
def handle_parse_item(self, response, item):
sel = Selector(response)
product_id_div = sel.xpath(
'//div[@id="storeCatalogEntryID"]/text()').extract()
if len(product_id_div) == 0 or len(product_id_div[0].strip()) == 0:
return
product_id = sel.xpath(
'//div[@id="storeCatalogEntryID"]/text()').extract()[0].strip()
item['show_product_id'] = product_id
sku_infos_str = sel.xpath('//div[@id="entitledItem_' + product_id +
'"]/text()').re(r'\[[\s\S]+\]')[0].strip()
context = execjs.compile('''
var sku_info = %s;
function getSkuInfo(){
return sku_info;
}
''' % sku_infos_str)
sku_infos = context.call('getSkuInfo')
dimensions = set([])
sizes = {}
colors = []
color_names = []
color_name_images_map = {}
color_list = sel.xpath('//ul[@class="detail_color"]/li')
if color_list:
for color_li in color_list:
color_item = Color()
color_item['show_product_id'] = product_id
color_item['from_site'] = self.name
color_item['type'] = 'color'
color_item['name'] = color_li.xpath(
'./a//div[@class="colorName"]/span/text()').extract(
)[0].strip()
color_names.append(color_item['name'])
colors.append(color_item)
else:
color_list = sel.xpath('//div[@class="color_swatch_list"]')[:-1]
if color_list:
for color_li in color_list:
for color_l in color_li.xpath('./ul/li'):
c = color_l.xpath('./a/@title').extract()[0]
color_item = Color()
color_item['show_product_id'] = product_id
color_item['from_site'] = self.name
color_item['type'] = 'color'
color_item['name'] = c
color_names.append(color_item['name'])
colors.append(color_item)
else:
color_item = Color()
color_item['name'] = 'one color'
color_names.append(color_item['name'])
colors.append(color_item)
sel.xpath('//ul[@id="thumb1"]/li')
skus = []
for sku_info in sku_infos:
sku_item = SkuItem()
sku_id = sku_info['catentry_id']
sku_item['id'] = sku_id
sku_item['type'] = 'sku'
sku_item['from_site'] = self.name
sku_item['show_product_id'] = product_id
attributes = sku_info['Attributes']
sku_size = {}
if attributes == {}:
attributes = {'vendorcolor_one color', 'size_one size'}
else:
for tempKey in attributes.keys():
if tempKey.find('Size') != -1:
temp = 1
break
else:
temp = 0
if temp == 0:
attributes['size_one size'] = '2'
for attribute in attributes:
keys = attribute.split('_')
dimension = keys[0].lower()
value = keys[1]
if dimension == 'vendorcolor':
dimension = 'color'
if value not in color_name_images_map.keys():
if 'ItemThumbnailImage' not in sku_info.keys():
return
thumbnail = sku_info['ItemThumbnailImage']
if not re.match(r'^http', thumbnail):
thumbnail = 'http:' + thumbnail
image = sku_info['ItemImage']
if not re.match(r'^http', image):
image = 'http:' + image
image = image + '&wid=970&hei=1245&fit=fit,1'
color_name_images_map[value] = {
'images': [{
'thumbnail': thumbnail,
'image': image
}]
}
color_cover = sku_info['ItemSwatchImage2']
if not re.match(r'^http', color_cover):
color_cover = 'http:' + color_cover
color_name_images_map[value]['cover'] = color_cover
dimensions.add(dimension)
sku_size[dimension] = value
if dimension != 'color':
if dimension not in sizes.keys():
sizes[dimension] = set([])
sizes[dimension].add(value)
sku_item['size'] = sku_size
if sku_info['offerPrice'] == '':
return
else:
sku_item['current_price'] = sku_info['offerPrice']
if sku_info['listPrice'] == '':
return
else:
sku_item['list_price'] = sku_info['listPrice']
sku_item['current_price'] = handle_price(sku_item['current_price'])
sku_item['list_price'] = handle_price(sku_item['list_price'])
if float(sku_item['list_price']) < float(
sku_item['current_price']):
sku_item['list_price'] = sku_item['current_price']
if sku_info['availableQuantity']:
sku_item['quantity'] = int(float(
sku_info['availableQuantity']))
else:
sku_item['quantity'] = 0
sku_item['is_outof_stock'] = sku_info['outOfStock']
if 'color' not in sku_item.keys():
sku_item['color'] = 'one color'
elif sku_item['color'] == {}:
sku_item['color'] = 'one color'
if 'size' not in sku_item.keys():
sku_item['size'] = 'one size'
elif sku_item['size'] == {}:
sku_item['size'] = 'one size'
skus.append(sku_item)
for color in colors:
if color['name'] in color_name_images_map.keys():
color['images'] = color_name_images_map[
color['name']]['images']
color['cover'] = color_name_images_map[color['name']]['cover']
yield color
item['dimensions'] = list(dimensions)
for size_key in sizes:
sizes[size_key] = list(sizes[size_key])
item['sizes'] = sizes
item['colors'] = color_names
item['skus'] = skus
item['desc'] = sel.xpath('//div[@id="detial_main_content"]').extract()
if item['desc']:
item['desc'] = item['desc'][0]
else:
item['desc'] = sel.xpath(
'//div[@class="descriptionsContent"]').extract()[0]
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
soldout_span = sel.xpath(
'//div[@class="cta-container"]//span[contains(@class, "soldout")]/text()'
).extract()
if len(soldout_span) > 0:
return
# siteObjStr = "".join(re.findall(r'(var siteObj = [\s\S]*exitting = false;)', response.body))
# context = execjs.compile('''
# %s
# function getSiteObj(){
# return siteObj;
# }
# ''' % siteObjStr)
# tmpStr = json.dumps(context.call('getSiteObj'))
# siteObj = json.loads(tmpStr)
# product_id = siteObj['productID']
# product_title = siteObj['productTitle']
# product_price = siteObj['productPrice']
# product_brand = siteObj['productBrand']
# product_rrp = siteObj['rrp']
product_id = eval(
re.search(r'productID:(.*),', response.body).group(1))
# product_title = eval(re.search(r'productTitle:(.*),', response.body).group(1))
product_price = eval(
re.search(r'productPrice:(.*),', response.body).group(1))
product_brand = eval(
re.search(r'productBrand:(.*),', response.body).group(1))
product_rrp = eval(re.search(r'rrp:(.*),', response.body).group(1))
if len(sel.xpath(".//h1[@itemprop='name']/text()")) > 0:
product_title = sel.xpath(
".//h1[@itemprop='name']/text()").extract()[0]
else:
current_url = response.url
product_title = ' '.join(current_url.split('/')[3].split('-'))
rrp_dr = re.compile(r'<[^>]+>', re.S)
rrp = rrp_dr.sub('', product_rrp)
rrp_prices = re.split(r': ', rrp)
if len(
sel.xpath(
".//*[@id='health-beauty']//div[@itemprop='description']").
extract()) > 0:
product_desc = sel.xpath(
".//*[@id='health-beauty']//div[@itemprop='description']"
).extract()[0]
else:
product_desc = ''
if product_id:
item['show_product_id'] = product_id
item['title'] = product_title
item['current_price'] = product_price.replace('£', '')
item['list_price'] = rrp_prices[1].replace('£', '')
item['desc'] = product_desc
item['brand'] = product_brand
item['dimensions'] = ['size']
item['sizes'] = ['onesize']
item['colors'] = ['onecolor']
if re.findall(r'men', item['brand']):
item['gender'] = 'men'
else:
return
thumbnails = sel.xpath(
".//ul[@class='product-thumbnails nav-items product-large-view-thumbs productImageZoom__thumbs']/li//div[contains(@class, 'product-thumb-box')]/img/@src"
).extract()
#thumbnails = sel.xpath(".//div[@class='product-thumb-box']/img/@src").extract()
# viewimages = sel.xpath(".//div[@class='product-thumb-box']/parent::*/@href").extract() #480*480
viewimages = sel.xpath(
".//ul[@class='product-thumbnails nav-items product-large-view-thumbs productImageZoom__thumbs']/li/a/@href"
).extract() #600*600
images = []
color_cover = None
if len(thumbnails) > 0:
imgKey = 0
for imgVal in thumbnails:
imageItem = ImageItem()
imageItem['image'] = handle_no_http(viewimages[imgKey].strip())
imageItem['thumbnail'] = handle_no_http(imgVal)
if not color_cover:
color_cover = imageItem['thumbnail']
images.append(imageItem)
imgKey += 1
color_form = sel.xpath(
'//div[@class="variation-dropdowns"]/form//option/@value').extract(
)[1:]
color_names = sel.xpath(
'//div[@class="variation-dropdowns"]/form//option/text()').extract(
)[1:]
size_or_color_str = sel.xpath(
'//div[@class="variation-dropdowns"]/form//label/text()').extract(
)
if len(size_or_color_str) > 0:
size_or_color = size_or_color_str[0]
if len(color_form) > 0 and size_or_color == 'Shade:':
item['colors'] = color_names
color_ajax_url = 'https://www.lookfantastic.com/variations.json?productId=' + str(
item['show_product_id'])
color_form_value = color_form.pop(0)
# for color_form_value in color_form:
form_data = {
'option1': str(color_form_value),
'selected': '1',
'variation1': '4'
}
# import pdb;pdb.set_trace()
yield FormRequest(color_ajax_url,
formdata=form_data,
callback=self.parse_color,
meta={
'item': item,
'color_forms': color_form
})
elif len(color_form) > 0 and size_or_color == 'Size:':
item['sizes'] = color_names
#pop color
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = 'onecolor'
colorItem['images'] = images
colorItem['cover'] = color_cover
yield colorItem
size_ajax_url = 'https://www.lookfantastic.com/variations.json?productId=' + str(
item['show_product_id'])
color_form_value = color_form.pop(0)
# for color_form_value in color_form:
form_data = {
'option1': str(color_form_value),
'selected': '1',
'variation1': '1'
}
# import pdb;pdb.set_trace()
yield FormRequest(size_ajax_url,
formdata=form_data,
callback=self.parse_size,
meta={
'item': item,
'color_forms': color_form
})
else:
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = product_id
colorItem['images'] = images
colorItem['cover'] = color_cover
colorItem['name'] = 'onecolor'
yield colorItem
item['skus'] = []
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = product_id
skuItem['id'] = product_id
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['from_site'] = item['from_site']
skuItem['color'] = 'onecolor'
skuItem['size'] = 'onesize'
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
# yield skuItem
yield item
def handle_parse_item(self, response, item):
if re.match(r'^http:\/\/us\.asos\.com\/mp_sp\/',response.url):
sel = Selector(response)
url = sel.xpath('//li[@id="mp_li_cnti"]/a/@href').extract()[0]
yield Request(url, callback=self.parse_item, cookies={'asos': 'currencyid=1'}, meta={'item': item})
else:
skus=[]
sel=Selector(response)
json_info = re.search("view\(\'(.+\})\'\,", response.body)
if not json_info:
return
else:
json_info = json_info.group(1)
json_info = "".join(json_info)
json_info = json_info.decode("string-escape")
goods_detail = json.loads(json_info)
descs = sel.xpath('//div[@class="overflow-container"]/div/div')
item['desc'] = ''
for desc in descs:
item['desc'] = item['desc'] + desc.extract()
item['title'] = goods_detail['name']
if 'brandName' not in goods_detail.keys():
item['brand'] = 'asos'
else:
item['brand'] = goods_detail['brandName']
item['from_site'] = self.name
if 'price' not in goods_detail.keys():
return
item['current_price'] = goods_detail['price']['current']
if float(goods_detail['price']['previous']) != 0:
item['list_price'] = goods_detail['price']['previous']
elif float(goods_detail['price']['rrp']) != 0:
item['list_price'] = goods_detail['price']['rrp']
else:
item['list_price'] = goods_detail['price']['current']
item['show_product_id'] = goods_detail['id']
sizes = []
colors = []
for sku in goods_detail['variants']:
skuItem = SkuItem()
skuItem['type'] = "sku"
skuItem['from_site'] = self.name
skuItem['is_outof_stock'] = False
skuItem['id'] = sku['variantId']
skuItem['show_product_id'] = goods_detail['id']
skuItem['current_price'] = item['current_price']
skuItem['list_price'] = item['list_price']
skuItem['size'] = sku['size']
if sku['size'] not in sizes:
sizes.append(sku['size'])
skuItem['color'] = sku['colour']
if sku['colour'] not in colors:
colors.append(sku['colour'])
skus.append(skuItem)
for color_name in colors:
images = []
for image in goods_detail['images']:
if image['colour'] == '' or (image['colour'] and color_name and len(image['colour']) == len(color_name) and (len(color_name) - difflib.SequenceMatcher(None,color_name,image['colour']).ratio()*len(color_name)) <=1):
imageItem = ImageItem()
imageItem['image'] = image['url'] + '?$XXL$'
imageItem['thumbnail'] = image['url']
images.append(imageItem)
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = goods_detail['id']
color['images'] = images
color['name'] = color_name
color['cover'] = images[0]['image']
yield color
item['skus'] = skus
item['sizes'] = list(set(sizes))
item['dimensions'] = ['size']
item['colors'] = colors
related_products_url = 'http://us.asos.com/api/product/catalogue/v2/productgroups/ctl/' + str(item['show_product_id']) + '?store=US&store=US¤cy=USD'
yield Request('http://us.asos.com/api/product/catalogue/v2/stockprice?productIds=' + str(goods_detail['id']) + '&store=US¤cy=USD', callback=self.parse_stock, meta={'item': item, 'related_products_url': related_products_url})
# color_size_str="".join(re.findall(r"var\s+arrSzeCol_ctl00_ContentMainPage_ctlSeparateProduct[^<]+", response.body))
# sep_image_str="".join(re.findall(r"var\s+arrSepImage_ctl00_ContentMainPage_ctlSeparateProduct[^<]+", response.body))
# thumb_image_str="".join(re.findall(r"var\s+arrThumbImage_ctl00_ContentMainPage_ctlSeparateProduct[^<]+", response.body))
# if len(color_size_str)>0:
# context = execjs.compile('''
# %s
# %s
# %s
# function get_color_size(){
# return arrSzeCol_ctl00_ContentMainPage_ctlSeparateProduct;
# }
# function get_sep_image(){
# return arrSepImage_ctl00_ContentMainPage_ctlSeparateProduct;
# }
# function get_thumb_image(){
# return arrThumbImage_ctl00_ContentMainPage_ctlSeparateProduct;
# }
# ''' % (color_size_str, sep_image_str, thumb_image_str))
# color_sizes = context.call('get_color_size')
# sep_image= context.call('get_sep_image')
# thumb_images = context.call('get_thumb_image')
# #import pdb;pdb.set_trace()
# if len(sel.xpath('//div[@id="ctl00_ContentMainPage_ctlSeparateProduct_pnlOutofStock"]').extract()) > 0:
# return
#
# if len(sel.xpath('//span[@id="ctl00_ContentMainPage_ctlSeparateProduct_lblProductTitle"]/text()').extract()) > 0:
# item['title']=sel.xpath('//span[@id="ctl00_ContentMainPage_ctlSeparateProduct_lblProductTitle"]/text()').extract()[0]
#
# data_dic_str = sel.xpath('//script[@id="dataDictionary"]/text()')
#
# product_data_str=data_dic_str.re(r'^var Product\s*=\s*({.*?});')[0]
# product_data=eval(product_data_str)
# item['show_product_id']=product_data['ProductIID']
# desc=sel.xpath('//div[@id="ctl00_ContentMainPage_productInfoPanel"]//ul')
# if len(desc)>0:
# item['desc']=desc.extract()[0]
# item['brand']=product_data['ProductBrand']
# item['from_site']=self.name
#
# '''有严重问题，注释掉了'''
# # gender_category_str=product_data['ProductCategory']
# # m=re.search(r'(.+)\|(.+)', gender_category_str)
# # if m:
# # item['gender']=m.group(1).strip()
# # m=re.search(r'(.+)\|(.+)', gender_category_str)
# # if m:
# # item['category']=m.group(2).strip()
#
# sku_data_str = data_dic_str.re(r'var ProductChildSkuInfo\s*=\s*({.*?});')[0]
# sku_data=eval(sku_data_str)
# sku_data_list=sku_data['ChildSkuInfo'][item['show_product_id']]
# #color_list=sel.xpath('//select[@id="ctl00_ContentMainPage_ctlSeparateProduct_drpdwnColour"]').extract()
# if color_sizes:
# '''handle color and image'''
#
# # thumbnail_lis=sel.xpath('//ul[@class="productThumbnails"]//li//img/@src')
# # image_lis=sel.xpath('//div[@id="productImages"]//img/@src')
# # if len(thumbnail_lis)>0:
# # for i in range(len(thumbnail_lis)):
# # imageItem=ImageItem()
# # imageItem['image']=image_lis[i].extract()
# # imageItem['thumbnail']=thumbnail_lis[i].extract()
# # images.append(imageItem)
# #left three imageItem
# images=[]
# for thumb_image in thumb_images:
# imageItem=ImageItem()
# imageItem['image']=thumb_image[2]
# imageItem['thumbnail']=thumb_image[0]
# images.append(imageItem)
#
# item_color_names=[]
# #all color names of item
#
# sep_image_dict = {}
# for sep_image_arr in sep_image:
# key = sep_image_arr[3]
# sep_image_dict[key] = {'image': sep_image_arr[2], 'thumbnail': sep_image_arr[0]}
#
# color_names = sel.xpath('//div[@id="ctl00_ContentMainPage_ctlSeparateProduct_pnlColour"]//option/@value')[1:].extract()
# for color_name in color_names:
#
# lower_color_name = color_name.lower()
# if '/' in lower_color_name:
# lower_color_name_2 = lower_color_name.replace('/', '')
# else:
# lower_color_name_2 = lower_color_name
# if lower_color_name not in sep_image_dict.keys() and lower_color_name_2 not in sep_image_dict.keys():
# return
# imageItem=ImageItem()
# imageItem['thumbnail']= sep_image_dict[lower_color_name_2]['thumbnail']
# imageItem['image']= sep_image_dict[lower_color_name_2]['image']
# images.insert(0, imageItem)
# # import pdb;pdb.set_trace()
# color=Color()
# color['type'] ='color'
# color['from_site'] = self.name
# color['show_product_id'] = product_data['ProductIID']
# color['images'] = images
# color['name'] = color_name
# color['cover'] = sep_image_dict[lower_color_name_2]['thumbnail']
#
# yield color
#
# item_color_names.append(color_name)
# '''handle price'''
# #list_price_sel=sel.xpath('//span[@id="ctl00_ContentMainPage_ctlSeparateProduct_lblRRP"]')
# sizes=[]
# for color_size in color_sizes:
# size_id = color_size[0]
# size = color_size[1]
# if not size.strip():
# size = 'onesize'
#
# if color_size[3] == "False":
# continue
#
# original_color_name = color_size[2]
# for color_name in item_color_names:
# tmp_color_name = re.sub(r'[^\w]', '', color_name)
#
# if tmp_color_name == original_color_name:
# original_color_name = color_name
#
# skuItem=SkuItem()
# skuItem['type']="sku"
# skuItem['from_site']=self.name
# skuItem['is_outof_stock']=False
# skuItem['id']=sku_data_list[str(size_id)+original_color_name]['Sku']
# #skuItem['id']=color_size[0]
# skuItem['show_product_id']=product_data['ProductIID']
# skuItem['current_price']= color_size[5]
#
# if color_size[6] == color_size[5] and color_size[8] != '0' and color_size[8] != '0.00':
# skuItem['list_price']= color_size[8]
# else:
# skuItem['list_price']= color_size[6]
#
# sizes.append(size)
# skuItem['color'] = original_color_name
# skuItem['size'] = size
# skus.append(skuItem)
#
# item['skus']=skus
# item['sizes']=list(set(sizes))
# item['dimensions']=['size']
# item['colors'] = item_color_names
# size_info = sel.xpath('//a[@id="ctl00_ContentMainPage_SizeGuideButton_SizeGuideLink"]/@href')
# if size_info:
# item['size_info'] = size_info.extract()[0]
# if not re.match(r'^http', size_info.extract()[0]):
# item['size_info'] = self.base_url + size_info.extract()[0]
# yield item
def handle_parse_item(self, response, item):
match = re.search(
r'<script type\=\"application\/json\">({"ProductDetails".+?)<\/script>',
response.body)
print match.group(1)
sel = Selector(response)
if match is None:
return
context = execjs.compile('''
var json = %s
function getJson(){
return json;
}
''' % match.group(1))
product_json = context.call('getJson')
main_product = product_json['ProductDetails']['main_products'][0]
item['brand'] = main_product['brand_name']['label']
item['title'] = main_product['short_description']
show_product_id = main_product['product_code']
item['show_product_id'] = show_product_id
item['desc'] = main_product['description']
list_price = main_product['price']['list_price']['usd_currency_value']
if re.findall('\-', list_price):
re.search('([\d\.]+)\s*\-', list_price).group(1)
else:
item['list_price'] = list_price
sale_price = main_product['price']['sale_price']['usd_currency_value']
if re.findall('\-', sale_price):
re.search('([\d\.]+)\s*\-', sale_price).group(1)
else:
item['current_price'] = sale_price
item['dimensions'] = ['size']
skus = []
sizes = {}
sizes['size'] = []
color_names = []
colors = main_product['colors']['colors']
handle_color_map = {}
if len(colors) > 0:
for color in colors:
handle_color_map[color['id']] = color['label']
handle_size_map = {}
if len(main_product['sizes']['sizes']) == 0:
sizes['size'].append('onesize')
else:
for size in main_product['sizes']['sizes']:
handle_size_map[size['id']] = size['value']
sizes['size'].append(size['value'])
image_prefix = 'http:' + main_product['media'][
'images_server_url'] + main_product['media']['images_path']
if len(colors) == 0:
color_name = 'onecolor'
color_names.append(color_name)
common_images = main_product['media']['images']
images = []
for common_image in common_images:
imageItem = ImageItem()
imageItem[
'image'] = image_prefix + common_image + '?wid=970&hei=1293&fmt=jpg'
imageItem[
'thumbnail'] = image_prefix + common_image + '?wid=396&hei=528&fmt=jpg'
images.append(imageItem)
first_thumbnail = images[0]['thumbnail']
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['images'] = images
colorItem['name'] = color_name
colorItem['cover'] = first_thumbnail
colorItem['version'] = '1'
yield colorItem
else:
common_images = main_product['media']['images']
for color in colors:
color_name = color['label']
color_names.append(color_name)
images = []
imageItem = ImageItem()
imageItem['image'] = image_prefix + color[
'colorize_image_url'] + '?wid=970&hei=1293&fmt=jpg'
imageItem['thumbnail'] = image_prefix + color[
'colorize_image_url'] + '?wid=396&hei=528&fmt=jpg'
images.append(imageItem)
first_thumbnail = images[0]['thumbnail']
for common_image in common_images:
imageItem = ImageItem()
imageItem[
'image'] = image_prefix + common_image + '?wid=970&hei=1293&fmt=jpg'
imageItem[
'thumbnail'] = image_prefix + common_image + '?wid=396&hei=528&fmt=jpg'
images.append(imageItem)
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['images'] = images
colorItem['name'] = color_name
colorItem['version'] = '1'
if len(color['value']) > 0:
if re.findall('\#', color['value']):
colorItem['cover_style'] = color['value']
else:
cover_img_str = sel.xpath(
'//li[@class="product-color-options__value" and @data-colorid='
+ str(color["id"]) + ']/@style').extract()
cover_unavi_str = sel.xpath(
'//li[@class="product-color-options__value product-color-options__value--unavailable" and @data-colorid='
+ str(color["id"]) + ']/@style').extract()
cover_sel_str = sel.xpath(
'//li[@class="product-color-options__value product-color-options__value--selected" and @data-colorid='
+ str(color["id"]) + ']/@style').extract()
cover_hid_str = sel.xpath(
'//li[@class="product-color-options__value is-hidden" and @data-colorid='
+ str(color["id"]) + ']/@style').extract()
if len(cover_img_str) > 0:
cover_img = re.search('\((.+)\)',
cover_img_str[0]).group(1)
colorItem['cover'] = 'http:' + cover_img
elif len(cover_unavi_str) > 0:
cover_img_str = cover_unavi_str[0]
cover_img = re.search('\((.+)\)',
cover_img_str).group(1)
colorItem['cover'] = 'http:' + cover_img
elif len(cover_sel_str) > 0:
cover_img_str = cover_sel_str[0]
cover_img = re.search('\((.+)\)',
cover_img_str).group(1)
colorItem['cover'] = 'http:' + cover_img
elif len(cover_hid_str) > 0:
cover_img_str = cover_hid_str[0]
cover_img = re.search('\((.+)\)',
cover_img_str).group(1)
colorItem['cover'] = 'http:' + cover_img
else:
colorItem['cover'] = first_thumbnail
else:
colorItem['cover'] = first_thumbnail
yield colorItem
item['colors'] = color_names
for sku in main_product['skus']['skus']:
sku_id = sku['sku_id']
if sku_id == 'DUMMY':
continue
if sku['color_id'] == -1:
color_name = 'onecolor'
else:
color_name = handle_color_map[sku['color_id']]
if sku['size_id'] == -1:
size = 'onesize'
else:
size = handle_size_map[sku['size_id']]
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['from_site'] = item['from_site']
skuItem['id'] = sku_id
skuItem['size'] = size
skuItem['color'] = color_name
if sku['status_alias'] == 'soldout' or sku[
'status_alias'] == 'waitlist':
skuItem['is_outof_stock'] = True
else:
skuItem['is_outof_stock'] = False
if len(sku['price']['sale_price']['usd_currency_value']) > 0:
skuItem['current_price'] = sku['price']['sale_price'][
'usd_currency_value']
else:
continue
if len(sku['price']['list_price']['usd_currency_value']) > 0:
skuItem['list_price'] = sku['price']['list_price'][
'usd_currency_value']
else:
continue
skus.append(skuItem)
item['sizes'] = sizes
item['skus'] = skus
if main_product['size_guide_link']['enabled'] == True:
sizeInfo = main_product['size_guide_link']['url']
findQ = sizeInfo.find("?")
if findQ != -1:
item['size_info'] = sizeInfo[:findQ]
else:
item['size_info'] = sizeInfo
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
if len(sel.xpath(".//div[@class='atg_store_noMatchingItem']")) > 0:
return
info = sel.xpath(".//div[@class='firstContainer row']")
item['brand'] = info.xpath("./h1/a[1]/text()").extract()[0]
item['show_product_id'] = info.xpath(
"./h1/h2/text()").extract()[0].strip()
item['title'] = info.xpath("./h1/a[2]/text()").extract()[0]
# item['desc'] = info.xpath("./h3/text()").extract()[0]
item['colors'] = []
if len(sel.xpath(".//div[@id='tab1_info']")) > 0:
if len(sel.xpath(".//div[@id='tab1_info']/div[2]")) > 0:
item['desc'] = sel.xpath(
".//div[@id='tab1_info']/div[1]/table").extract(
)[0] + sel.xpath(
".//div[@id='tab1_info']/div[2]/table").extract()[0]
else:
item['desc'] = sel.xpath(
".//div[@id='tab1_info']/div[1]/table").extract()[0]
skusStr = "".join(
re.findall(r'window.universal_variable =.+\}\}<\/script>',
response.body, re.S))
if len(skusStr) > 0:
context = execjs.compile('''
var skus = %s;
function getSkus(){
return skus;
}
''' % skusStr[27:-9])
skusDict = context.call('getSkus')
item['list_price'] = skusDict['product']['unit_price']
item['current_price'] = skusDict['product']['unit_sale_price']
images = []
imageDom = sel.xpath(".//ul[@class='alt_imgs col-md-12']/li")
colorItem = Color()
for dom in imageDom:
imageItem = ImageItem()
imageItem['image'] = self.base_url + dom.xpath(
"./a/@href").extract()[0]
imageItem['thumbnail'] = re.sub('XA\.', 'LA.', imageItem['image'])
images.append(imageItem.copy())
colorItem['images'] = images
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = u'one color'
colorItem['cover'] = self.base_url + sel.xpath(
".//ul[@class='alt_imgs col-md-12']/li[1]/a/img/@src").extract()[0]
yield colorItem
item['colors'].append(colorItem['name'])
item['dimensions'] = ['size']
item['skus'] = []
sku_item_url_list = []
sku_size_list = []
index = 0
sku_items = sel.xpath(".//div[@id='sizeValues']/div")
if len(sku_items) > 0:
for sku_item in sku_items:
sku_size = sku_item.xpath("./@onclick").extract()[0].split(
"'")[3]
ajax_id = sku_item.xpath("./@onclick").extract()[0].split(
"'")[1]
if sku_size.find(' ') != -1:
sku_size = re.sub(' ', '%20', sku_size)
sku_item_url = self.base_url + sel.xpath(
".//form[@id='colorsizerefreshform']/@action"
).extract(
)[0] + '&productId=' + ajax_id + '&selectedSize=' + sku_size
sku_item_url_list.append(sku_item_url)
sku_size_list.append(sku_size)
sku_item_url_list.append(
sku_item_url
) # only for avoiding indexError in parse_sku_item when loop reach the last size
yield Request(sku_item_url_list[0],
callback=self.parse_sku_item,
meta={
"sku_size_list": sku_size_list,
"sku_item_url_list": sku_item_url_list,
"item": item,
"index": index
})
else:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = skusDict['product']['id']
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['color'] = u'one color'
skuItem['size'] = u'one size'
skuItem['id'] = skusDict['product']['sku_code']
skuItem['from_site'] = item['from_site']
if skusDict['product']['stock'] == 0:
skuItem['is_outof_stock'] = True
item['skus'].append(skuItem)
item['sizes'] = [u'one size']
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
if 'show_product_id' not in item.keys():
item['show_product_id'] = sel.xpath(
".//input[@id='baseNo']/@value").extract()[0]
retailPrice = re.sub(
r'\s', '',
sel.xpath(".//span[@class='retailPrice']/a/text()").extract()
[0])
find = retailPrice.find('$')
item['list_price'] = retailPrice[find + 1:]
item['desc'] = sel.xpath(
".//div[@id='overviewSection']/div[1]/*").extract()[0]
if len(sel.xpath(
".//div[@class='sizingChart']/table/*").extract()) > 0:
item['desc'] += '<div><table>'
for d in sel.xpath(".//div[@class='sizingChart']/table/*"):
item['desc'] += d.extract()
item['desc'] += '</table></div>'
item['dimensions'] = []
item['sizes'] = {}
item['colors'] = []
propertyDropDown = sel.xpath(".//div[@class='propertyDropDown']")
dropDownValue = {}
skuKey = {}
if len(propertyDropDown.extract()):
largeImage = sel.xpath(
".//input[@id='largeImageSrcTemplate']/@value").extract()[0]
zoomImage = sel.xpath(
".//input[@id='zoomImageHrefTemplate']/@value").extract()[0]
i = 1
for dropDown in propertyDropDown:
title = (sel.xpath(".//label[@id='property" + str(i) +
"Label']/@title").extract()[0]).lower()
skuKey['property' + str(i)] = title
vk = {}
dom = dropDown.xpath("./select/option[@value != '']")
for d in dom:
k = d.xpath("./@value").extract()[0]
v = d.xpath("./text()").extract()[0]
vk[k] = v
if title == 'color':
imageItem = ImageItem()
colorItem = Color()
altImages = sel.xpath(".//a[@class='altImage']")
prod_images = []
imageItem['thumbnail'] = re.sub(
r'_\d+~\d+', '_' + k + '~460', largeImage)
imageItem['image'] = re.sub(r'_\d+~\d+',
'_' + k + '~1500',
zoomImage)
colorItem['cover'] = re.sub(r'_\d+~\d+',
'_' + k + '~220',
imageItem['thumbnail'])
prod_images.append(imageItem.copy())
for altImage in altImages:
temp_image = altImage.xpath("./@href").extract()[0]
temp_thumbnail = altImage.xpath(
"./@data-image").extract()[0]
for temp_dict in prod_images:
t = 0
if temp_dict[
'thumbnail'] == temp_thumbnail or imageItem[
'image'] == temp_image:
t = -1
break
if t != -1:
imageItem['image'] = temp_image
imageItem['thumbnail'] = temp_thumbnail
prod_images.append(imageItem.copy())
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['images'] = prod_images
colorItem['name'] = v + '(' + k + ')'
item['colors'].append(colorItem['name'])
yield colorItem
dropDownValue[title] = vk
i += 1
if title == 'color':
pass
else:
item['dimensions'].append(title)
item['sizes'][title] = vk.values()
if len(item['colors']) == 0:
return
if len(item['dimensions']) == 0:
item['dimensions'] = ['size']
if 'size' in item['sizes'].keys():
pass
else:
item['sizes']['size'] = ['onesize']
item['cover'] = imageItem['thumbnail'].replace("460", "220")
skusStr = "".join(
re.findall(r'[\s]*var skus = new TAFFY\([\s]*(\[.*\])[\s]*\);',
response.body, re.S))
if skusStr:
context = execjs.compile('''
var skus = %s;
function getSkus(){
return skus;
}
''' % skusStr)
skusDict = context.call('getSkus')
item['skus'] = []
for sku in skusDict:
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = sku['finalPromoPrice'][1:]
skuItem['color'] = dropDownValue['color'][
sku['property1']] + '(' + sku['property1'] + ')'
if sku['property2'] == '':
skuItem['size'] = {'size': 'onesize'}
skuItem['id'] = sku['property1']
else:
skuItem['size'] = {
skuKey['property2']:
dropDownValue[skuKey['property2']][sku['property2']]
}
skuItem['id'] = sku['property1'] + '*' + sku['property2']
if sku['property3'] != '':
skuItem['size'][skuKey['property3']] = dropDownValue[
skuKey['property3']][sku['property3']]
skuItem['id'] += '*' + sku['property3']
skuItem['from_site'] = item['from_site']
skuItem['is_outof_stock'] = False
if 'size' in skuItem.keys():
if 'size' in skuItem['size'].keys():
pass
else:
skuItem['size']['size'] = 'onesize'
else:
skuItem['size'] = {'size': 'onesize'}
item['skus'].append(skuItem)
yield item
def handle_parse_item(self, response, item):
if 'Error' in response.url:
return
sel = Selector(response)
item['category'] = sel.xpath(
'//section[@id="Breadcrumbs"]/nav/a[last()]/text()').extract()[0]
show_product_id = sel.xpath('//span[@id="Sku"]/text()').extract()[-1]
details_str = ''.join(
re.search('sf\.productDetail\.init\((\{[\s\S]+?\})\)\;',
response.body).group(1))
context = execjs.compile('''
var skus = %s;
function getSkus(){
return skus;
}
''' % details_str)
item['skus'] = []
details = context.call('getSkus')
skus_json = details['variants']
images_json = details['images']
images = []
sizes = []
width = []
dimensions_dict = {}
is_in_stock = False
for children in details['children']:
if children['c'] == show_product_id:
color_name = children['v']
is_in_stock = True
if not is_in_stock:
return
list_price = ''.join(
sel.xpath(
'//div[@id="Price"]/div[@class="floatLeft productPriceInfo"]/span[@class="origPrice"]//span/text()'
).extract()).strip()
current_price = ''.join(
sel.xpath(
'//div[@id="Price"]/span[@class="floatLeft productPrice "]/text() | //div[@id="Price"]/span[@class="floatLeft productPrice "]/span/text()'
).extract()).strip()
if not list_price:
list_price = ''.join(
sel.xpath(
'//div[@id="Price"]//span/text()').extract()).strip()
current_price = list_price
for sku_json in skus_json:
skuItem = {}
skuItem['type'] = 'sku'
skuItem['id'] = sku_json['v']
skuItem['show_product_id'] = show_product_id
skuItem['list_price'] = list_price
skuItem['current_price'] = current_price
skuItem['from_site'] = item['from_site']
skuItem['is_outof_stock'] = False
skuItem['size'] = {}
for sku in sku_json['a']:
if sku['n'] == 'custom.color':
skuItem['color'] = sku['v']
else:
if sku['n'] not in dimensions_dict.keys():
dimensions_dict[sku['n']] = []
if sku['v'] not in dimensions_dict[sku['n']]:
dimensions_dict[sku['n']].append(sku['v'])
skuItem['size'][sku['n']] = sku['v']
# if sku['n'] == 'size':
# skuItem['size']['size'] = sku['v']
# dimensions_dict['size'].append(sku['v'])
# if sku['n'] == 'width':
# skuItem['size']['width'] = sku['v']
# dimensions_dict['width'].append(sku['v'])
if skuItem['size']['size'] not in sizes:
sizes.append(skuItem['size']['size'])
if 'width' in skuItem['size'].keys(
) and skuItem['size']['width'] not in width:
width.append(skuItem['size']['width'])
item['skus'].append(skuItem)
for img_json in images_json:
if img_json['c'] == show_product_id:
imageItem = ImageItem()
imageItem['thumbnail'] = img_json['f']
imageItem['image'] = img_json[
'f'][:img_json['f'].find('?')] + '?wid=1280&hei=900'
images.append(imageItem)
if not images:
return
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = show_product_id
colorItem['images'] = images
colorItem['cover'] = images[0]['thumbnail']
colorItem['name'] = color_name
yield colorItem
item['show_product_id'] = show_product_id
item['dimensions'] = dimensions_dict.keys()
if len(sel.xpath('//div[@id="Description"]/p/text()')) > 0:
item['desc'] = sel.xpath(
'//div[@id="Description"]/p/text()').extract()[0].strip()
else:
item['desc'] = ''.join(
sel.xpath('//ul[@class="productFeaturesDetails"]//li').extract(
)).strip()
item['colors'] = [color_name]
item['sizes'] = dimensions_dict
yield item
def parse(self, response):
goods_all = json.loads(self.json_data)
skus = goods_all['skus']
goods_colors = goods_all['goods_colors']
goods = goods_all['goods']
if 'title' not in goods.keys():
raise NotSupported('No title in goods, Pls specify!!')
title = ''.join(goods['title'].split()).lower()
make_md5 = hashlib.md5()
make_md5.update(quote(title, safe=''))
show_product_id = make_md5.hexdigest()
logging.warning('show_product_id: ' + show_product_id)
logging.warning('title: ' + goods['title'])
from_site = goods['from_site']
skus_colors = []
skus_sizes = []
for sku in skus:
if sku['color'] not in skus_colors:
skus_colors.append(sku['color'])
skus_sizes.append(sku['size'])
if 'goods_current_price' not in dir(
) or goods_current_price > sku['current_price']:
goods_current_price = sku['current_price']
if 'goods_list_price' not in dir(
) or goods_list_price < sku['list_price']:
goods_list_price = sku['list_price']
sku['type'] = 'sku'
sku['id'] = show_product_id + '-' + sku['color'] + '-' + sku['size']
sku['is_outof_stock'] = False
sku['from_site'] = from_site
sku['show_product_id'] = show_product_id
goods_colors_colors = []
for goods_color in goods_colors:
if 'name' not in goods_color.keys():
raise NotSupported('ERROR! No name in (color)')
goods_colors_colors.append(goods_color['name'])
if 'cover' not in goods.keys():
goods['cover'] = goods_color['images'][0]['image']
if 'images' not in goods_color.keys() or len(
goods_color['images']) == 0 or (
len(goods_color['images']) == 1
and len(goods_color['images'][0]) == 0):
goods_color['images'] = [{'image': goods['cover']}]
goods_color['cover'] = goods['cover']
else:
goods_color['cover'] = goods_color['images'][0]['image']
goods_color['from_site'] = from_site
goods_color['show_product_id'] = show_product_id
# if goods_colors_colors != list(set(goods_colors_colors)):
# raise NotSupported('ERROR! Dupelicate name in goods_colors')
if goods_colors_colors == [] or skus_colors == [] or sorted(
skus_colors) != sorted(goods_colors_colors):
raise NotSupported('skus_colors: ' + str(skus_colors) + ', ' +
'goods_colors_colors: ' +
str(goods_colors_colors) + ', \n' +
'goods_colors not equal sku_colors')
for goods_color in goods_colors:
colorItem = Color()
colorItem = goods_color
colorItem['type'] = 'color'
yield colorItem
item = BaseItem()
item = goods
item['type'] = 'base'
item['product_type'] = 'json_import_' + goods['product_type_id']
item['category'] = 'json_import_' + goods['category_id']
item['product_type_id'] = goods['product_type_id']
item['category_id'] = goods['category_id']
item['title'] = goods['title']
item['show_product_id'] = show_product_id
item['from_site'] = from_site
item['colors'] = skus_colors
item['sizes'] = skus_sizes
item['current_price'] = goods_current_price
item['list_price'] = goods_list_price
if 'groupbuy_num' in goods.keys():
item['groupbuy_num'] = goods['groupbuy_num']
item['skus'] = skus
print 'skus', skus
if 'desc' in goods.keys():
item['desc'] = goods['desc']
if 'weight' in goods.keys():
item['weight'] = goods['weight']
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
item['current_price'] = product_price[6:]
item['list_price'] = rrp_prices[1][6:]
if 'show_product_id' not in item.keys():
item['show_product_id'] = eval(re.search(r'productID:(.*),', response.body).group(1))
if 'brand' not in item.keys():
item['brand'] = eval(re.search(r'productBrand:(.*),', response.body).group(1))
if item['brand'] == '':
item['brand'] = 'allsole'
desc = sel.xpath(".//div[@itemprop='description']/*").extract()
if len(desc) > 0:
item['desc'] = desc[0]
else:
item['desc'] = ''
sizeDoms = sel.xpath(".//select[@id='opts-1']/option[string-length(@value) > 0]")
item['dimensions'] = ['size']
item['skus'] = []
if len(sizeDoms.extract()):
colorDoms = sel.xpath(".//select[@id='opts-2']/option[@value>0]")
if len(colorDoms.extract()):
color = colorDoms.xpath("./text()").extract()[0]
item["colors"] = [color]
else:
item["colors"] = ["onecolor"]
item['sizes'] = []
for dom in sizeDoms:
curr_size = dom.xpath("./text()").extract()[0]
item['sizes'].append(curr_size)
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = item["colors"][0] + '*' + curr_size
skuItem['color'] = item["colors"][0]
skuItem['size'] = curr_size
##skuItem['size_num'] = re.search(r'\d+', curr_size).group(0)
skuItem['from_site'] = self.name
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
else:
item['sizes'] = ['onesize']
item['colors'] = ['onecolor']
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = item['colors'][0] + '*' + item['sizes'][0]
skuItem['color'] = item['colors'][0]
skuItem['size'] = item['sizes'][0]
skuItem['from_site'] = self.name
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
imgDoms = sel.xpath(".//div[@class='media']//li[@class='n-unit']")
images = []
if len(imgDoms.extract()):
for dom in imgDoms:
imageItem = ImageItem()
imageItem['image'] = dom.xpath("./a/@href").extract()[0]
imageItem['thumbnail'] = dom.xpath("./a/div/img/@src").extract()[0]
images.append(imageItem)
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = self.name
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = item['colors'][0]
colorItem['cover'] = images[0]['thumbnail']
colorItem['images'] = images
yield colorItem
yield item
def parse_color(self, response):
item = response.meta['item']
color_forms = response.meta['color_forms']
# print color_forms
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
response_body = re.sub(r'null', 'None', response.body)
color_form_response = eval(response_body)
if 'images' in color_form_response.keys():
thumbnail = color_form_response['images'][0]['name']
if len(color_form_response['images']) < 4:
return
image = color_form_response['images'][3]['name']
if '/600' not in image:
image = re.sub('/[\d]+/[\d]+', '/600/600', image)
color_name = color_form_response['variations'][0]['options'][0][
'name']
cover_style = color_form_response['variations'][0]['options'][0][
'value']
price_str = color_form_response['price']
m = re.search(r'\d+\.\d+', price_str)
if m:
current_price = m.group(0)
imageItem = ImageItem()
imageItem['thumbnail'] = 'https://s4.thcdn.com/' + thumbnail
imageItem['image'] = 'https://s4.thcdn.com/' + image
images = []
images.append(imageItem)
colorItem['name'] = color_name
colorItem['images'] = images
colorItem['cover_style'] = cover_style
yield colorItem
if 'skus' not in item.keys():
item['skus'] = []
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = color_name + '*' + 'onesize'
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = current_price
skuItem['from_site'] = item['from_site']
skuItem['color'] = color_name
skuItem['size'] = 'onesize'
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
if len(color_forms) == 0:
yield item
else:
# import pdb;pdb.set_trace()
color_ajax_url = 'https://www.lookfantastic.com/variations.json?productId=' + str(
item['show_product_id'])
color_form_value = color_forms.pop(0)
# for color_form_value in color_form:
form_data = {
'option1': str(color_form_value),
'selected': '1',
'variation1': '4'
}
# import pdb;pdb.set_trace()
yield FormRequest(color_ajax_url,
formdata=form_data,
callback=self.parse_color,
meta={
'item': item,
'colorItem': colorItem,
'color_forms': color_forms
})
def handle_parse_item(self, response, item):
sel = Selector(response)
if len(
sel.xpath(
".//div[@class='field mb20 mt20 detail-info-outStock']").
extract()) > 0 or len(
sel.xpath(
".//div[@class='soldOut color-red bold h4']")) > 0:
return
if len(sel.xpath(".//span[@itemprop='sku']/text()")) == 0:
return
if 'show_product_id' not in item.keys():
item['show_product_id'] = sel.xpath(
".//span[@itemprop='sku']/text()").extract()[0]
item['desc'] = sel.xpath(
".//div[@data-tstid='Content_Composition&Care']/dl").extract()[0]
item['dimensions'] = ['size', 'color']
if len(
sel.xpath(".//a[@data-target='.sizeGuideModal']/@href").
extract()) > 0:
item['size_info'] = self.base_url + sel.xpath(
".//a[@data-target='.sizeGuideModal']/@href").extract()[0]
product_json = re.search(
'window\.universal_variable\.product[\s]*=[\s]*(.+)[\s]*</script>',
response.body).group(1)
product_json = unicode(product_json, errors='replace')
if len(product_json) > 0:
context = execjs.compile('''
var skus = %s;
function getDetail(){
return skus;
}
''' % product_json)
goods_detail = context.call('getDetail')
categoryId = goods_detail['categoryId']
designerId = goods_detail['manufacturerId']
productId = goods_detail['id']
storeId = goods_detail['storeId']
if goods_detail['color'] == "":
color_name = "One Color"
else:
color_name = goods_detail['color']
list_price = goods_detail['unit_price']
current_price = goods_detail['unit_sale_price']
else:
return
images = []
image_lis = sel.xpath(
'.//ul[@class="sliderProduct js-sliderProduct js-sliderProductPage"]/li'
)
for image_li in image_lis:
imageItem = ImageItem()
imageItem['image'] = image_li.xpath(
'./a/img/@data-zoom-image').extract()[0]
imageItem['thumbnail'] = image_li.xpath(
'./a/img/@data-large').extract()[0]
images.append(imageItem)
if images == []:
return
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = item['show_product_id']
color['images'] = images
color['name'] = color_name
cover = re.sub('_480\.jpg', '_70.jpg', images[0]['thumbnail'])
color['cover'] = cover
yield color
# data={'categoryId': str(categoryId), 'designerId': str(designerId), 'productId': str(productId), 'storeId': str(storeId)}
# print data
# {'storeId': '9446', 'categoryId': '135967', 'designerId': '4981', 'productId': '11497594'}
# # url = 'https://us-il.proxymesh.com:31280'
# # username = '******'
# # password = '******'
# # password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
# # auth_handler = urllib2.HTTPBasicAuthHandler(password_mgr)
# # opener = urllib2.build_opener(auth_handler)
# # # urllib2.install_opener(opener)
# #
# # # opener = urllib2.build_opener()
# # opener.addheaders.append(('Cookie', 'ckm-ctx-sf=/'))
# # sku_resp = opener.open('http://www.farfetch.com/product/GetDetailState/', urllib.urlencode(data))
# # sku_resp_str = sku_resp.read()
# # sku_detail_str = json.loads(sku_resp_str)['Sizes'].strip()
# if len(sku_resp_str) > 0:
# context = execjs.compile('''
# var skus = %s;
# function getDetail(){
# return skus;
# }
# ''' % sku_resp_str)
# sku_detail = context.call('getDetail')
# print sku_detail
detail_url = 'http://www.farfetch.com/product/GetDetailState?' + 'categoryId=' + str(
categoryId) + '&designerId=' + str(
designerId) + '&productId=' + str(
productId) + '&storeId' + str(storeId)
yield Request(detail_url,
callback=self.parse_sku,
meta={
'item': item,
'color_name': color_name,
'list_price': list_price,
'current_price': current_price
},
cookies={'ckm-ctx-sf': '/'},
dont_filter=True)
def handle_parse_item(self, response, item):
sel = Selector(response)
product_str = re.search('var dataLayer\s*=\s*(.+?);',
response.body).group(1)
product_json = json.loads(product_str)
product = product_json['products'][0]
item['brand'] = 'Tiffany'
item['title'] = product['name']
if not product['price']:
return
item['list_price'] = product['price']
item['current_price'] = product['price']
item['desc'] = sel.xpath(
'//div[@id="drawerDescription"]/div/div').extract()[0]
item['cover'] = sel.xpath('//meta[@property="og:image"]/@content'
).extract()[0] + self.cover_img_surffix
if product['stockStatus'] == 'out of stock':
return
skus = []
sizes = []
if not sel.xpath(
'//select[@id="ctlSkuGroupType1_selItemList"]/option'):
item['show_product_id'] = product['sku']
skuItem = SkuItem()
skuItem['type'] = "sku"
skuItem['from_site'] = item['from_site']
skuItem['size'] = 'One Size'
sizes = ['One Size']
skuItem['color'] = 'One Color'
skuItem['id'] = item['show_product_id'] + '-' + skuItem[
'color'] + '-' + skuItem['size']
skuItem['show_product_id'] = item['show_product_id']
skuItem['current_price'] = item['current_price']
skuItem['list_price'] = item['list_price']
skus.append(skuItem)
else:
item['show_product_id'] = product['groupSku']
size_options = sel.xpath(
'//select[@id="ctlSkuGroupType1_selItemList"]/option')
for size_option in size_options:
skuItem = SkuItem()
skuItem['type'] = "sku"
skuItem['from_site'] = item['from_site']
if not size_option.xpath('./text()').extract():
skuItem['size'] = 'One Size'
else:
skuItem['size'] = size_option.xpath(
'./text()').extract()[0]
sizes.append(skuItem['size'])
skuItem['color'] = 'One Color'
skuItem['id'] = item['show_product_id'] + '-' + skuItem[
'color'] + '-' + skuItem['size']
skuItem['show_product_id'] = item['show_product_id']
skuItem['current_price'] = item['current_price']
skuItem['list_price'] = item['list_price']
skus.append(skuItem)
images = []
imageItem = ImageItem()
imageItem['thumbnail'] = sel.xpath(
'//meta[@property="og:image"]/@content').extract(
)[0] + self.large_img_surffix
imageItem['image'] = sel.xpath('//meta[@property="og:image"]/@content'
).extract()[0] + self.large_img_surffix
images.append(imageItem)
color = Color()
color['type'] = 'color'
color['from_site'] = item['from_site']
color['show_product_id'] = item['show_product_id']
color['images'] = images
color['name'] = 'One Color'
color['cover'] = item['cover']
yield color
item['colors'] = ['One Color']
item['sizes'] = sizes
item['skus'] = skus
item['dimensions'] = ['size']
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
item['show_product_id'] = re.search('(\d+)\.html',
response.url).group(1)
item['title'] = sel.xpath(
'//div[@class="product-name"]/h1/text()').extract()[0].strip()
brand = re.search('brand: \'(.+)\'', response.body)
if not brand:
item['brand'] = 'pharmacyonline'
else:
item['brand'] = brand.group(1)
img = re.search('imgUrl: \'(.+)\'', response.body).group(1)
images = []
imageItem = ImageItem()
imageItem[
'thumbnail'] = img + '?imageMogr2/thumbnail/380x380/extent/380x380/background/d2hpdGU='
imageItem['image'] = img
images.append(imageItem)
# item['cover'] = images[0]['thumbnail']
item['colors'] = ['One Color']
color = Color()
color['type'] = 'color'
color['from_site'] = item['from_site']
color['show_product_id'] = item['show_product_id']
color['images'] = images
color['name'] = 'One Color'
color['cover'] = images[0][
'image'] + '?imageMogr2/thumbnail/100x100/extent/100x100/background/d2hpdGU='
yield color
item['desc'] = sel.xpath(
'//div[@class="product-collateral"]').extract()[0]
current_price = sel.xpath(
'//div[@class="DetailNoDis PriceNow last_price_sing"]/span/text()')
if len(current_price) > 0:
item['current_price'] = current_price.extract()[0]
item['list_price'] = item['current_price']
else:
item['current_price'] = sel.xpath(
'//div[@class="DetailPriceContain clearfix"]//div[@class="PriceNow"]/text()'
).extract()[0].strip()
item['list_price'] = sel.xpath(
'//div[@class="DetailPriceContain clearfix"]//p[@class="PriceWas"]/text()'
).extract()[0].strip()
skus = []
item['sizes'] = ['One Size']
skuItem = SkuItem()
skuItem['type'] = "sku"
skuItem['from_site'] = item['from_site']
sku_id = sel.xpath(
'//div[@class="DetailSku"]/text()').extract()[0].strip()
skuItem['id'] = re.search('(\d+)', sku_id).group(1)
skuItem['show_product_id'] = item['show_product_id']
skuItem['current_price'] = item['current_price']
skuItem['list_price'] = item['list_price']
skuItem['size'] = 'One Size'
skuItem['color'] = 'One Color'
skus.append(skuItem)
item['skus'] = skus
item['dimensions'] = ['size']
if len(item['show_product_id']) > 6:
product_id = item['show_product_id'][1:]
else:
product_id = item['show_product_id']
stock_url = 'http://cn.pharmacyonline.com.au/pt_catalog/index/checkQty?product_id=' + product_id
yield Request(stock_url,
callback=self.parse_stock,
meta={"item": item},
dont_filter=True)
def handle_parse_item(self, response, baseItem):
product_detail_str="".join(re.findall(r"var\s+productDetail[^;]+", response.body))
if len(product_detail_str)>0:
context = execjs.compile('''
%s
function get_product_detail(){
return productDetail;
}
''' % (product_detail_str))
product_detail = context.call('get_product_detail')
sel = Selector(response)
product_id = sel.xpath('//div[@id="productId"]/text()').extract()[0]
skus = []
baseItem['from_site'] = self.name
baseItem['show_product_id'] = product_id
size_js_infos = product_detail['sizes']
size_infos = {}
size_values = []
for size_id in size_js_infos:
size_infos[size_js_infos[size_id]['sizeCode']] = size_id
size_values.append(size_id)
list_price = sel.xpath('//div[@id="productPrices"]//meta[@itemprop="price"]/@content').extract()[0]
color_price_blocks = sel.xpath('//div[@id="productPrices"]//div[@class="priceBlock"]')
color_price_mapping = {}
for color_price_block in color_price_blocks:
color_name = color_price_block.xpath('./span[@class="priceColors"]/text()').extract()
if len(color_name) > 0:
regular_price_span = color_price_block.xpath('./span[@class="regularPrice"]/text()').extract()
if len(regular_price_span) > 0:
color_price_mapping[color_name[0]] = regular_price_span[0]
else:
color_price_mapping[color_name[0]] = color_price_block.xpath('./span[@class="salePrice"]/text()').extract()[0]
image_items = product_detail['colors']
color_names = []
for key in image_items:
imageItems = image_items[key]['images']
color_name = image_items[key]['colorName'].strip()
color_names.append(color_name)
images=[]
tmp_images = []
for image_key in imageItems:
imageItem = ImageItem()
image = imageItems[image_key]
imageItem['thumbnail'] = image['thumbnail']
imageItem['image'] = image['zoom']
tmp_images.append((image['index'], imageItem))
tmp_images = sorted(tmp_images, key=lambda x:x[0])
for tmp_tuple in tmp_images:
images.append(tmp_tuple[1])
colorItem = Color()
colorItem['type'] = 'color'
colorItem['show_product_id'] = product_id
colorItem['from_site'] = self.name
colorItem['cover'] = image_items[key]['swatch']
colorItem['name'] = color_name
colorItem['images'] = images
yield colorItem
sizes = image_items[key]['sizes']
for size in sizes:
size_name = size_infos[size]
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['from_site'] = self.name
skuItem['color'] = color_name
skuItem['show_product_id'] = product_id
skuItem['id'] = key+"-"+size
skuItem['size'] = size_name
skuItem['list_price'] = list_price
if len(color_price_mapping)>0 and color_name in color_price_mapping.keys():
# skuItem['current_price'] = sale_price_span.re(r'\d+.?\d*')[0]
skuItem['current_price'] = color_price_mapping[colorItem['name']]
else:
skuItem['current_price'] = skuItem['list_price']
skuItem['is_outof_stock'] = False
skus.append(skuItem)
baseItem['sizes'] = size_values
baseItem['colors']= color_names
baseItem['skus'] = skus
size_fit_container = sel.xpath('//div[@id="sizeFitContainer"]')
if len(size_fit_container)>0:
size_fit = size_fit_container.extract()[0]
baseItem['desc'] = '<div>'+sel.xpath('//div[@itemprop="description"]').extract()[0]+size_fit+"</div>"
else:
baseItem['desc'] = sel.xpath('//div[@itemprop="description"]').extract()[0]
baseItem['dimensions'] = ['size', 'color']
yield baseItem
def handle_parse_item(self, response, item):
sel = Selector(response)
product_id_var = re.search(r'var pr_page_id[\s]*=[\s]*([^;]+);',
response.body)
if not product_id_var:
return
# if len(sel.xpath('//div[@class="outOfStock outOfStockSpecial"]')) > 0:
# return
product_id = eval(product_id_var.group(1))
item['show_product_id'] = product_id
templateid = sel.xpath(
'//div[@id="templateOption"]/@templateid').extract()
if len(templateid) > 0:
templateid = templateid[0]
color_rows = sel.xpath('//tr[contains(@class, "diaperItemTR")]')
clothing_shoes_products = sel.xpath(
'//div[contains(@class, "clothingShoesProducts")]')
if templateid == '6':
''''''
colorItems = {}
colorNames = []
oneColorDiv = sel.xpath('//div[@id="oneSelection"]')
if len(oneColorDiv.extract()) > 0:
colorItem = Color()
color_cover = oneColorDiv.xpath(".//img/@src").extract()[0]
if re.match(r'^//', color_cover):
color_cover = 'https:' + color_cover
color_name = oneColorDiv.xpath("text()").extract()[0]
colorItem["cover"] = color_cover
colorItem["name"] = color_name
colorItem['type'] = 'color'
colorItem['from_site'] = self.from_site
colorItems[color_name] = {"item": colorItem, "handled": False}
colorNames.append(color_name)
else:
colorDivs = sel.xpath('//ul[@id="falvorDrownList"]/li')
#colorDivs = sel.xpath('//div[contains(@class, "clothingShoesProducts")]/div[contains(@class, "clothProductItem")]//div[contains(@class, "colorPaneItems")]');
for colorDiv in colorDivs:
colorItem = Color()
color_cover = colorDiv.xpath(".//img/@src").extract()[0]
if re.match(r'^//', color_cover):
color_cover = 'https:' + color_cover
color_name = colorDiv.xpath("@id").extract()[0]
colorItem["cover"] = color_cover
colorItem["name"] = color_name
colorItem['type'] = 'color'
colorItem['from_site'] = self.from_site
colorItems[color_name] = {
"item": colorItem,
"handled": False
}
colorNames.append(color_name)
skuHiddens = sel.css('.multiItemBox').xpath(
'.//input[@class="skuHidden"]'
) #sel.xpath('//div[@id="clothItem"]//input[@class="skuHidden"]')
skus = []
sizes = []
if len(skuHiddens) > 0:
for skuHidden in skuHiddens:
if skuHidden.xpath('@value').extract()[0] != "":
skuItem = SkuItem()
skuItem['show_product_id'] = item['show_product_id']
regular_price = skuHidden.xpath(
'@regularprice').extract()
price = skuHidden.xpath('@price').extract()[0]
skuid = skuHidden.xpath('@value').extract()[0]
is_outof_stock = skuHidden.xpath(
'@isoutofstock').extract()[0]
skuItem['id'] = skuid
if len(regular_price) > 0 and regular_price[0] != '':
skuItem["list_price"] = regular_price[0]
else:
skuItem["list_price"] = price
skuItem['current_price'] = price
replace_skuid = skuid.replace('-', '_')
size = sel.xpath(
'//ul[@id="diaperItemTR' + replace_skuid +
'"]//li[@class="itemSize"]/span/text()').extract(
)[0].strip()
color_name = sel.xpath('//ul[@id="diaperItemTR' +
replace_skuid +
'"]/@primaryattr').extract()[0]
skuItem['size'] = size
if not size in sizes:
sizes.append(size)
skuItem['color'] = color_name
skuItem[
'is_outof_stock'] = self.change_out_of_stock_str(
is_outof_stock)
skuItem['type'] = 'sku'
skuItem['from_site'] = self.from_site
#yield skuItem
skus.append(skuItem)
if colorItems[color_name]["handled"] == False:
colorItems[color_name]["handled"] = True
url = 'https://www.diapers.com/product/productDetail!GetSkuImage.qs?skuCode=' + skuid + '&random=' + '%f' % random(
)
yield Request(url,
meta={
'item':
colorItems[color_name]['item'],
'show_product_id':
item['show_product_id']
},
callback=self.parse_image)
item["skus"] = skus
item['sizes'] = sizes
item['colors'] = colorNames
elif len(color_rows.extract()) > 0:
'''只有一个尺寸表格'''
skus = []
colorNames = []
for color_row in color_rows:
colorItem = Color()
colorItem['from_site'] = self.from_site
colorItem['type'] = 'color'
skuHidden = color_row.xpath('.//input[@class="skuHidden"]')
skuItem = SkuItem()
skuItem['show_product_id'] = item['show_product_id']
regular_price = skuHidden.xpath('@regularprice').extract()
price = skuHidden.xpath('@price').extract()[0]
skuid = skuHidden.xpath('@value').extract()[0]
is_outof_stock = skuHidden.xpath('@isoutofstock').extract()[0]
skuItem['id'] = skuid
# print response.url
# print is_outof_stock
# print skuHidden.extract()
if len(regular_price) > 0 and regular_price[0] != '':
skuItem["list_price"] = regular_price[0]
else:
skuItem["list_price"] = price
skuItem['current_price'] = price
color_cover = color_row.xpath(
'td[@class="itemImage"]/span[contains(@class, "itemImageDiv")]/img/@src'
).extract()[0]
if re.match(r'^//', color_cover):
color_cover = 'https:' + color_cover
colorItem['cover'] = color_cover
color_name = color_row.xpath(
'td[contains(@class, "Description")]/text()').extract()
if len(color_name) > 0:
color_name = color_name[0].strip()
else:
color_name = color_row.xpath(
'td[contains(@class, "elseDescription")]/text()'
).extract()
if len(color_name) > 0:
color_name = color_name[0].strip()
else:
color_name = color_row.xpath(
'td[contains(@class, "itemDescription")]/text()'
).extract()
if len(color_name) > 0:
color_name = color_name[0].strip()
if not color_name:
color_name = 'onecolor'
colorItem['name'] = color_name
colorNames.append(color_name)
skuItem['color'] = color_name
skuItem['size'] = 'onesize'
skuItem['is_outof_stock'] = self.change_out_of_stock_str(
is_outof_stock)
skuItem['type'] = 'sku'
skuItem['from_site'] = self.from_site
#yield skuItem
skus.append(skuItem)
url = 'https://www.diapers.com/product/productDetail!GetSkuImage.qs?skuCode=' + skuid + '&random=' + '%f' % random(
)
yield Request(url,
meta={
'item': colorItem,
'show_product_id': item['show_product_id']
},
callback=self.parse_image)
item["skus"] = skus
item['sizes'] = ['onesize']
item['colors'] = colorNames
elif len(clothing_shoes_products) > 0:
'''最常见的格式'''
colorDivs = sel.xpath(
'//div[contains(@class, "clothingShoesProducts")]/div[contains(@class, "clothProductItem")]//div[contains(@class, "colorPaneItems")]'
)
colorItems = {}
colorNames = []
for colorDiv in colorDivs:
colorItem = Color()
if len(colorDiv.xpath('./img').extract()) > 0:
color_cover = colorDiv.xpath("./img/@src").extract()[0]
if re.match(r'^//', color_cover):
color_cover = 'https:' + color_cover
color_name = colorDiv.xpath("./img/@color").extract()[0]
colorItem["cover"] = color_cover
elif len(colorDiv.xpath("./div/@style").extract()) > 0:
cover_style = colorDiv.xpath("./div/@style").extract()[0]
color_name = colorDiv.xpath("./div/@color").extract()[0]
if 'background:' in cover_style:
cover_style = re.search('background:([^;]+)',
cover_style).group(1)
colorItem["cover_style"] = cover_style
else:
return
colorItem["name"] = color_name
colorItem['type'] = 'color'
colorItem['from_site'] = self.from_site
colorItems[color_name] = {"item": colorItem, "handled": False}
colorNames.append(color_name)
skuHiddens = sel.xpath(
'//div[@id="clothItem"]//input[@class="skuHidden"]')
skus = []
sizes = []
if len(skuHiddens) > 0:
for skuHidden in skuHiddens:
if skuHidden.xpath('@value').extract()[0] != "":
skuItem = SkuItem()
skuItem['show_product_id'] = item['show_product_id']
regular_price = skuHidden.xpath(
'@regularprice').extract()
price = skuHidden.xpath('@price').extract()[0]
skuid = skuHidden.xpath('@value').extract()[0]
is_outof_stock = skuHidden.xpath(
'@isoutofstock').extract()[0]
skuItem['id'] = skuid
if len(regular_price) > 0 and regular_price[0] != '':
skuItem["list_price"] = regular_price[0]
else:
skuItem["list_price"] = price
skuItem['current_price'] = price
size = sel.xpath(
'//div[@id="clothItem"]//input[@sku="' + skuid +
'"]/@value').extract()[0]
color_name = sel.xpath(
'//div[@id="clothItem"]//input[@sku="' + skuid +
'"]/@primaryattributevalue').extract()[0]
skuItem['size'] = size
if not size in sizes:
sizes.append(size)
skuItem['color'] = color_name
skuItem[
'is_outof_stock'] = self.change_out_of_stock_str(
is_outof_stock)
skuItem['type'] = 'sku'
skuItem['from_site'] = self.from_site
#yield skuItem
skus.append(skuItem)
if colorItems[color_name]["handled"] == False:
colorItems[color_name]["handled"] = True
url = 'https://www.diapers.com/product/productDetail!GetSkuImage.qs?skuCode=' + skuid + '&random=' + '%f' % random(
)
yield Request(url,
meta={
'item':
colorItems[color_name]['item'],
'show_product_id':
item['show_product_id']
},
callback=self.parse_image)
item["skus"] = skus
item['sizes'] = sizes
item['colors'] = colorNames
elif len(
sel.xpath(
'//div[@id="templateOption"]/div[contains(@class, "colorSizeFirstStep")]//li[contains(@class, "colorPaneItems")]'
).extract()) > 0:
'''判断是否是只有颜色没有尺寸的情况'''
colorDivs = sel.xpath(
'//div[@id="templateOption"]/div[contains(@class, "colorSizeFirstStep")]//li[contains(@class, "colorPaneItems")]'
)
colorItems = {}
colorNames = []
skus = []
for colorDiv in colorDivs:
colorItem = Color()
if len(colorDiv.xpath('./img').extract()) > 0:
color_cover = colorDiv.xpath("./img/@src").extract()[0]
if re.match(r'^//', color_cover):
color_cover = 'https:' + color_cover
color_name = colorDiv.xpath("./img/@color").extract()[0]
colorItem["cover"] = color_cover
else:
cover_style = colorDiv.xpath("./div/@style").extract()[0]
color_name = colorDiv.xpath("./div/@color").extract()[0]
if 'background:' in cover_style:
cover_style = re.search('background:([^;]+)',
cover_style).group(1)
colorItem["cover_style"] = cover_style
colorItem["name"] = color_name
colorItem['type'] = 'color'
colorItem['from_site'] = self.from_site
colorItems[color_name] = {"item": colorItem, "handled": False}
colorNames.append(color_name)
skuid = colorDiv.xpath('./img/@sku').extract()[0]
skuHidden = sel.xpath('//input[@id="skuHidden' +
skuid.replace('-', '_') + '"]')
skuItem = SkuItem()
skuItem['show_product_id'] = item['show_product_id']
regular_price = skuHidden.xpath('@regularprice').extract()
price = skuHidden.xpath('@price').extract()[0]
is_outof_stock = skuHidden.xpath('@isoutofstock').extract()[0]
skuItem['id'] = skuid
if len(regular_price) > 0 and regular_price[0] != '':
skuItem["list_price"] = regular_price[0]
else:
skuItem["list_price"] = price
skuItem['current_price'] = price
size = 'onesize'
skuItem['size'] = size
skuItem['color'] = color_name
skuItem['is_outof_stock'] = self.change_out_of_stock_str(
is_outof_stock)
skuItem['type'] = 'sku'
skuItem['from_site'] = self.from_site
#yield skuItem
skus.append(skuItem)
url = 'https://www.diapers.com/product/productDetail!GetSkuImage.qs?skuCode=' + skuid + '&random=' + '%f' % random(
)
yield Request(url,
meta={
'item': colorItem,
'show_product_id': item['show_product_id']
},
callback=self.parse_image)
item['sizes'] = ['onesize']
item["skus"] = skus
item['colors'] = colorNames
elif len(
sel.xpath(
'//div[@id="templateOption"]/div[contains(@class, "colorSizeFirstStep")]'
).extract()) > 0:
colorItem = Color()
# print sel.xpath('//div[@id="QtyInputDiv"]//li[contains(@class,"itemImage")]').extract()
# color_cover = sel.xpath('//div[@id="QtyInputDiv"]//li[contains(@class,"itemImage")]//img/@src').extract()[0]
color_name = 'onecolor'
# colorItem['cover'] = color_cover
colorItem['name'] = color_name
colorItem['type'] = 'color'
colorItem['from_site'] = self.from_site
sku_sizes = sel.css('.colorSizeFirstStep .collectionSelections'
).xpath('./li/input')
skus = []
sizes = []
colorItemSku = ''
for sku_size in sku_sizes:
''''''
skuItem = SkuItem()
skuItem['show_product_id'] = item['show_product_id']
skuid = sku_size.xpath('@sku').extract()[0]
skuHidden = sel.xpath('//input[@id="skuHidden' +
skuid.replace('-', '_') + '"]')
regular_price = skuHidden.xpath('@regularprice').extract()
price = skuHidden.xpath('@price').extract()[0]
is_outof_stock = skuHidden.xpath('@isoutofstock').extract()[0]
skuItem['id'] = skuid
if len(regular_price) > 0 and regular_price[0] != '':
skuItem["list_price"] = regular_price[0]
else:
skuItem["list_price"] = price
skuItem['current_price'] = price
size = sel.xpath('//img[@id="' + skuid.replace('-', '_') +
'ColorButton"]/@colorvalue').extract()[0]
skuItem['size'] = size
if not size in sizes:
sizes.append(size)
skuItem['color'] = color_name
skuItem['is_outof_stock'] = self.change_out_of_stock_str(
is_outof_stock)
skuItem['type'] = 'sku'
skuItem['from_site'] = self.from_site
#yield skuItem
if colorItemSku == '':
colorItemSku = skuid
color_cover = sel.xpath(
'//img[@id="' + skuid.replace('-', '_') +
'ColorButton"]/@imgsrc').extract()[0]
if re.match(r'^//', color_cover):
color_cover = 'https:' + color_cover
colorItem['cover'] = color_cover
skus.append(skuItem)
item["skus"] = skus
item['sizes'] = sizes
item["colors"] = ['onecolor']
url = 'https://www.diapers.com/product/productDetail!GetSkuImage.qs?skuCode=' + colorItemSku + '&random=' + '%f' % random(
)
yield Request(url,
meta={
'item': colorItem,
'show_product_id': item['show_product_id']
},
callback=self.parse_image)
elif templateid == '10':
# primary_attr = sel.xpath('//table[@id="primaryAttributeList"]/@attributename')
# primary_attr = primary_attr.extract()[0].strip().lower() if len(primary_attr)>0 else ''
#
# second_attr = sel.xpath('//table[@id="secondAttributeList"]/@attributename')
# second_attr = second_attr.extract()[0].strip().lower() if len(second_attr)>0 else ''
#
# third_attr = sel.xpath('//table[@id="thirdAttributeList"]/@attributename')
# third_attr = third_attr.extract()[0].strip().lower() if len(third_attr)>0 else ''
#
# item['dimensions'] = [dimension for dimension in [primary_attr,second_attr,third_attr] if dimension]
#
# primary_attr_ids = sel.xpath('//table[@id="primaryAttributeList"]/tr/@id').extract()
# second_attr_ids = sel.xpath('//table[@id="secondAttributeList"]/tr/@id').extract()
# third_attr_ids = sel.xpath('//table[@id="thirdAttributeList"]/tr/@id').extract()
#
# primary_attr_dict = {}
# second_attr_dict = {}
# third_attr_dict = {}
# item['sizes'] = {}
skus = []
# if primary_attr and len(primary_attr_ids) > 0:
# item['sizes'][primary_attr] = []
# for primary_attr_id in primary_attr_ids:
# primary_attr_value = sel.xpath('//table[@id="primaryAttributeList"]/tr[@id="' + str(primary_attr_id) + '"]/td[@class="attributeValue "]/b/text()').extract()[0]
# item['sizes'][primary_attr].append(primary_attr_value)
# primary_attr_dict[primary_attr_id] = primary_attr_value
#
# if second_attr and len(second_attr_ids) > 0:
# item['sizes'][second_attr] = []
# for second_attr_id in second_attr_ids:
# second_attr_value = sel.xpath('//table[@id="secondAttributeList"]/tr[@id="' + str(second_attr_id) + '"]/td[@class="attributeValue "]/b/text()').extract()[0]
# item['sizes'][second_attr].append(second_attr_value)
# second_attr_dict[second_attr_id] = second_attr_value
#
# if third_attr and len(third_attr_ids) > 0:
# item['sizes'][third_attr] = []
# for third_attr_id in third_attr_ids:
# third_attr_value = sel.xpath('//table[@id="thirdAttributeList"]/tr[@id="' + str(third_attr_id) + '"]/td[@class="attributeValue "]/b/text()').extract()[0]
# item['sizes'][third_attr].append(third_attr_value)
# third_attr_dict[third_attr_id] = third_attr_value
# if 'sizes' not in item['sizes'].keys():
# item['sizes']['size'] = ['One Size']
# if 'color' not in item['sizes'].keys():
# item['colors'] = ['One Color']
product_json_str = re.search('var pdpOptionsJson\s*=\s*([^\n]+);',
response.body)
if product_json_str:
product_json_str = product_json_str.group(1)
product_json = json.loads(product_json_str)
colors = []
for sku_json in product_json:
skuItem = SkuItem()
skuItem['id'] = sku_json['Sku']
skuItem['show_product_id'] = item['show_product_id']
skuItem['size'] = {}
skuItem['color'] = sku_json['Description']
skuItem['size'] = 'One Size'
colors.append(skuItem['color'])
# if primary_attr and len(primary_attr_dict) > 0 and len(sku_json['PrimaryAttributeValue'])>0:
# skuItem['size'][primary_attr] = primary_attr_dict[sku_json['PrimaryAttributeValue']]
#
# if second_attr and len(second_attr_dict) > 0 and len(sku_json['SecondAttributeValue'])>0:
# skuItem['size'][second_attr] = second_attr_dict[sku_json['SecondAttributeValue']]
#
# if third_attr and len(third_attr_dict) > 0 and len(sku_json['ThirdAttributeValue'])>0:
# skuItem['size'][third_attr] = third_attr_dict[sku_json['ThirdAttributeValue']]
# if len(skuItem['size']) == 0:
# skuItem['size'] = 'One Size'
# skuItem['color'] = 'One Color'
# else:
# if 'size' not in skuItem['size'].keys():
# skuItem['size'] = 'One Size'
# if 'color' not in skuItem['size'].keys():
# skuItem['color'] = 'One Color'
skuItem['current_price'] = sku_json['RetailPrice']
if not sku_json['RegularPrice']:
skuItem["list_price"] = sku_json['RetailPrice']
else:
skuItem['list_price'] = sku_json['RegularPrice']
skuItem['is_outof_stock'] = self.change_out_of_stock_str(
sku_json['IsOutOfStock'])
skuItem['type'] = 'sku'
skuItem['from_site'] = self.from_site
skus.append(skuItem)
colorItem = Color()
colorItem['name'] = skuItem['color']
colorItem['type'] = 'color'
colorItem['from_site'] = self.from_site
color_url = 'https://www.diapers.com/product/productDetail!GetSkuImage.qs?skuCode=' + skuItem[
"id"] + '&random=' + '%f' % random()
yield Request(color_url,
meta={
'item': colorItem,
'show_product_id': item['show_product_id']
},
callback=self.parse_image)
item['skus'] = skus
item['colors'] = colors
item['sizes'] = ['One Size']
else:
'''告警'''
# print response.meta['url']
return
desc = sel.css('.descriptTabContent').xpath(
"//div[@class='pIdDesContent']").extract()
if len(desc) > 0:
item['desc'] = desc[0]
else:
item['desc'] = ''
'''handle size info'''
size_chart_type = re.search(r'var sizeChartType[\s]*=[\s]*"([^"]+)";',
response.body)
size_info_brand_name = re.search(r'var brandName[\s]*=[\s]*"([^"]+)";',
response.body)
if size_chart_type and size_info_brand_name:
size_info_brand_name = size_info_brand_name.group(1)
size_chart_type = size_chart_type.group(1)
item['size_info'] = {
'brand_name': size_info_brand_name,
'size_chart_type': size_chart_type
}
'''size info chart url'''
size_info_chart_url = 'https://www.diapers.com/Product/BrandSizeChartHopup.qs?brandName=' + quote(
size_info_brand_name) + '&sizeChartType=' + quote(
size_chart_type)
#print size_info_chart_url
yield Request(size_info_chart_url,
meta={
'brand_name': size_info_brand_name,
'size_chart_type': size_chart_type
},
callback=self.parse_size_info)
yield item
def handle_parse_item(self, response, baseItem):
sel = Selector(response)
product_id = sel.xpath('//div[@id="productId"]/text()').extract()[0]
baseItem['gender'] = 'men'
baseItem['type'] = 'base'
baseItem['from_site'] = self.name
baseItem['show_product_id'] = product_id
baseItem['title'] = sel.xpath(
'//span[@class="row product-title"]/text()').extract()[0].strip()
size_fit_container = sel.xpath('//div[@id="sizeFitContainer"]')
if len(size_fit_container) > 0:
size_fit = size_fit_container.extract()[0]
baseItem['desc'] = '<div>' + sel.xpath(
'//div[@itemprop="description"]').extract(
)[0] + size_fit + "</div>"
else:
baseItem['desc'] = sel.xpath(
'//div[@itemprop="description"]').extract()[0]
baseItem['dimensions'] = ['size', 'color']
skus = []
product_detail_str = "".join(
re.findall(r"var\s+productDetail[^;]+", response.body))
if len(product_detail_str) > 0:
context = execjs.compile('''
%s
function get_product_detail(){
return productDetail;
}
''' % (product_detail_str))
product_detail = context.call('get_product_detail')
size_js_infos = product_detail['sizes']
size_infos = {}
size_values = []
for size_id in size_js_infos:
size_infos[size_js_infos[size_id]['sizeCode']] = size_id
size_values.append(size_id)
list_price = sel.xpath(
'//div[@id="productPrices"]//meta[@itemprop="price"]/@content'
).extract()[0]
color_price_blocks = sel.xpath(
'//div[@id="productPrices"]//div[@class="priceBlock"]')
# color_price_mapping = {}
# for color_price_block in color_price_blocks:
# color_name = color_price_block.xpath(
# './span[@class="priceColors"]/text()').extract()
# if len(color_name) > 0:
# regular_price_span = color_price_block.xpath(
# './span[@class="regularPrice"]/text()').extract()
# if len(regular_price_span) > 0:
# color_price_mapping[color_name[0]] = regular_price_span[0]
# else:
# color_price_mapping[color_name[0]] = color_price_block.xpath(
# './span[@class="salePrice"]/text()').extract()[0]
match = re.search(r'productPage\.sellingPrice\=\'([\d\.]+)\';',
response.body)
if match is None:
current_price = list_price
else:
current_price = match.group(1)
image_items = product_detail['colors']
color_names = []
for key in image_items:
imageItems = image_items[key]['images']
color_name = image_items[key]['colorName'].strip() + '-' + str(key)
color_names.append(color_name)
images = []
tmp_images = []
for image_key in imageItems:
imageItem = ImageItem()
image = imageItems[image_key]
imageItem['thumbnail'] = image['thumbnail']
imageItem['image'] = image['zoom']
tmp_images.append((image['index'], imageItem))
tmp_images = sorted(tmp_images, key=lambda x: x[0])
for tmp_tuple in tmp_images:
images.append(tmp_tuple[1])
colorItem = Color()
colorItem['type'] = 'color'
colorItem['show_product_id'] = baseItem['show_product_id']
colorItem['from_site'] = self.name
colorItem['cover'] = image_items[key]['swatch']
colorItem['name'] = color_name
colorItem['images'] = images
yield colorItem
sizes = image_items[key]['sizes']
for size in sizes:
size_name = size_infos[size]
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['from_site'] = self.name
skuItem['color'] = color_name
skuItem['show_product_id'] = baseItem['show_product_id']
skuItem['id'] = key + "-" + size
skuItem['size'] = size_name
skuItem['list_price'] = list_price
skuItem['current_price'] = current_price
# if len(color_price_mapping) > 0 and color_name in color_price_mapping.keys():
# skuItem['current_price'] = color_price_mapping[
# colorItem['name']]
# else:
# skuItem['current_price'] = skuItem['list_price']
skuItem['is_outof_stock'] = False
skus.append(skuItem)
baseItem['sizes'] = size_values
baseItem['colors'] = color_names
baseItem['skus'] = skus
product_items = sel.xpath(
'//ul[@id="similarities"]/li[@class="product"]')
if len(product_items) > 0:
related_items_id = []
for product_item in product_items:
product_id = product_item.xpath(
'./div/div[@class="info"]/img/@data-product-id').extract(
)[0]
related_items_id.append(product_id)
if related_items_id:
baseItem['related_items_id'] = related_items_id
yield baseItem
def handle_parse_item(self, response, item):
skus = []
sel = Selector(response)
item['from_site'] = self.name
if 'whoops' in response.url:
logging.warning('anti scraping: ' + response.url)
match = re.search(r'"product_id":\s*\[\s*"([^"]+)"\s*\]',
response.body)
if match is None:
return
temp_show_product_id = match.group(1)
current_price = sel.xpath(
'//span[contains(@class, "price-sales")]/text()').extract()
if len(current_price) > 0:
current_price = current_price[0]
list_price = sel.xpath(
'//span[contains(@class, "price-standard")]/text()').re(
r'(\S+)')
if len(list_price) > 0:
list_price = list_price[0]
else:
list_price = current_price
item['brand'] = self.name
# item['desc']=".".join(sel.xpath('//div[contains(@class, "additional")]/ul/li/text()').extract())
desc1 = sel.xpath(
'//div[@class="categorylisting detail"]/div/div').extract()[0]
desc2 = sel.xpath(
'//div[@class="categorylisting fabric"]/div/div').extract()[0]
item['desc'] = re.sub(r'[\t\n]', '', desc1 + desc2)
item['desc'] = re.sub('<img.+?>', '', item['desc'])
if sel.xpath(
'//div[contains(@class, "quantity clearfix")]//p[contains(@class, "in-stock-msg")]/text()'
):
colors = []
item_colors_links = sel.xpath(
'//div[@id="product-content"]//ul[contains(@class, "swatches color")]//li[contains(@class,"selected")]/a'
)
item_sizes = sel.xpath(
'//div[@id="product-content"]//div[contains(@class, "value")]//ul[contains(@class, "swatches size")]/li[@class!="emptyswatch unselectable"]//@title'
).extract()
item['sizes'] = item_sizes
item['dimensions'] = ['size']
item['product_type'] = 'mother-baby'
if len(item_colors_links) == 0:
item_colors_links = ['one_color']
for item_color_link in item_colors_links:
images = []
thumbnails = sel.xpath(
'//div[@id="thumbnails"]//li[@class!="thumb pdpvideo"]'
)
if thumbnails:
for li in thumbnails:
#thumbnails_evl=li.xpath('./a/img/@src').extract()[0]
imageItem = ImageItem()
image_url = li.xpath('./a/img/@src').extract()[0]
imageItem['image'] = self.handle_image_url(
image_url.encode('utf-8'), 1000, 1000)
imageItem['thumbnail'] = self.handle_image_url(
image_url.encode('utf-8'), 350, 350)
images.append(imageItem)
elif sel.xpath(
'//div[@id="thumbnails"]/li[@class="thumb pdpvideo"]/a/img/@src'
):
imageItem = ImageItem()
image_url = sel.xpath(
'//img[@class="primary-image"]/@src').extract()[0]
imageItem['image'] = self.handle_image_url(
image_url.encode('utf-8'), 1000, 1000)
imageItem['thumbnail'] = self.handle_image_url(
image_url.encode('utf-8'), 350, 350)
images.append(imageItem)
else:
imageItem = ImageItem()
image_url = sel.xpath(
'//img[@class="primary-image"]/@src').extract()[0]
imageItem['image'] = self.handle_image_url(
image_url.encode('utf-8'), 1000, 1000)
imageItem['thumbnail'] = self.handle_image_url(
image_url.encode('utf-8'), 350, 350)
images.append(imageItem)
if len(item_colors_links) > 0:
color_name = item_color_link.xpath(
'./@title').extract()[0]
color_cover = item_color_link.xpath('./@style').re(
'http://[^\)]+')[0]
else:
color_name = 'one_color'
color_cover = images[0]['thumbnail']
colors.append(color_name)
show_product_id = temp_show_product_id + "*" + color_name
item['show_product_id'] = show_product_id
color = Color()
color['type'] = 'color'
color['from_site'] = self.name
color['show_product_id'] = show_product_id
color['images'] = images
color['name'] = color_name
color['cover'] = color_cover
yield color
for item_size in item_sizes:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = show_product_id
skuItem[
'id'] = item['show_product_id'] + "*" + item_size
skuItem['current_price'] = current_price
skuItem['list_price'] = list_price
if len(item_colors_links) > 0:
skuItem['color'] = item_color_link.xpath(
'./@title').extract()[0]
else:
skuItem['color'] = 'one_color'
skuItem['size'] = item_size
skuItem['from_site'] = self.name
skuItem['is_outof_stock'] = False
skuItem['quantity'] = sel.xpath(
'//select[contains(@id, "Quantity")]//@value'
).extract()[0]
#yield skuItem
skus.append(skuItem)
item['colors'] = colors
item['skus'] = skus
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
if len(sel.xpath('//h1[@class="pageTitle"]/text()')) > 0:
return
if len(sel.xpath('//section[@class="productDesc"]')) > 0:
return
"""单品s"""
if len(
sel.xpath(
'//div[@class="productQuickDetails"]//p[@class="productPrice sale textSale"]/span/text()'
)) > 0:
item['current_price'] = sel.xpath(
'//div[@class="productQuickDetails"]//p[@class="productPrice sale textSale"]/span/text()'
).extract()[0]
item['list_price'] = sel.xpath(
'//div[@class="productQuickDetails"]//p[@class="productPrice regular"]/span/text()'
).extract()[0]
else:
item['current_price'] = sel.xpath(
'//div[@class="productQuickDetails"]//p[@class="productPrice "]/span/text()'
).extract()[0]
item['list_price'] = sel.xpath(
'//div[@class="productQuickDetails"]//p[@class="productPrice "]/span/text()'
).extract()[0]
if '-' in item['list_price']:
item['list_price'] = item['list_price'].split('-')[-1].strip()
if '-' in item['current_price']:
item['current_price'] = item['current_price'].split(
'-')[-1].strip()
item['show_product_id'] = sel.xpath(
'//div[@class="productNumber"]/text()').extract()[0]
if 'Item' in item['show_product_id']:
item['show_product_id'] = re.search(
'(\d+)', item['show_product_id']).group(1)
item['desc'] = sel.xpath(
".//div[@class='productShortDescription']/p/text()").extract()
if len(sel.xpath('//div[@class="longDescription"]')) > 0:
item['desc'] = item['desc'] + sel.xpath(
'//div[@class="longDescription"]').extract()
if len(sel.xpath('div[@class="productSpecs"]')) > 0:
item['desc'] = item['desc'] + sel.xpath(
'div[@class="productSpecs"]').extract()
item['desc'] = ''.join(item['desc'])
variants_json_str = re.search('var variantsJson = (.+);',
response.body).group(1)
variants_json = json.loads(variants_json_str)
item['skus'] = []
item['sizes'] = []
if not variants_json['items']:
item['sizes'] = ['One Size']
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = re.search('productId: \"(\d+)\"',
response.body).group(1)
skuItem['color'] = 'One Color'
skuItem['size'] = 'One Size'
skuItem['from_site'] = self.name
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['is_outof_stock'] = False
item['skus'].append(skuItem)
else:
for variant_json in variants_json['items']:
item['sizes'].append(variant_json['id'])
skuItem = {}
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = variant_json['catEntryId']
skuItem['color'] = 'One Color'
skuItem['size'] = variant_json['id']
skuItem['from_site'] = self.name
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = variant_json['price']
skuItem['is_outof_stock'] = not variant_json['buyable']
item['skus'].append(skuItem)
if 'sizeChart' in variants_json['variants'][0].keys():
item['size_chart'] = 'https:' + variants_json['variants'][0][
'sizeChart']
img_as = sel.xpath(".//a[@class='productThumb thumbnailImage']")
images = []
if len(img_as) > 0:
for img_a in img_as:
imageItem = ImageItem()
imageItem['image'] = img_a.xpath("./img/@src").extract()[0]
imageItem['thumbnail'] = img_a.xpath("./img/@src").extract()[0]
if '?' in imageItem['image']:
imageItem['image'] = imageItem['image'].split(
'?')[0] + self.image_url_suffix
if '?' in imageItem['thumbnail']:
imageItem['thumbnail'] = imageItem['thumbnail'].split(
'?')[0] + self.cover_url_suffix
images.append(imageItem)
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = self.name
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = 'One Color'
colorItem['cover'] = images[0]['thumbnail']
colorItem['images'] = images
yield colorItem
item['colors'] = ['One Color']
if 'size_chart' in item.keys():
yield Request(item['size_chart'],
callback=self.parse_size_chart,
meta={"item": item},
dont_filter=True)
else:
yield item
def handle_parse_item(self, response, item):
body_json = json.loads(response.body)
goods_detail = body_json['data']
if goods_detail['inStock'] == 0:
return
item['linkhaitao_url'] = response.url
item['cover'] = goods_detail['coverImgUrl']
item['desc'] = goods_detail['content']['description']
if 'product_type_id' in os.environ.keys():
self.product_type_id = os.environ['product_type_id']
if 'category_id' in os.environ.keys():
self.category_id = os.environ['category_id']
if self.product_type_id:
item['product_type_id'] = int(self.product_type_id)
item['product_type'] = 'linkhaitao_' + str(self.product_type_id)
if self.category_id:
item['category_id'] = int(self.category_id)
item['category'] = 'linkhaitao_' + str(self.category_id)
if 'editor_flag' in os.environ.keys():
self.editor_flag = os.environ['editor_flag']
if self.editor_flag:
item['editor_flag'] = self.editor_flag
if 'gender' in os.environ.keys():
self.gender = os.environ['gender']
if self.gender:
item['gender'] = self.gender
item['dimensions'] = ['size', 'color']
item['brand'] = goods_detail['brand']['name_en']
item['title'] = goods_detail['name']
item['current_price'] = goods_detail['realPriceOrg']
item['list_price'] = goods_detail['mallPriceOrg']
from_site = ''.join(goods_detail['sellerName']['namecn'].split()).lower()
if self.is_chinese_word(from_site):
from_site = ''.join(goods_detail['sellerName']['namecn'].split()).lower()
if "'" in from_site:
from_site = from_site.replace("'", "")
if '/' in from_site:
from_site = from_site.split('/')[0]
item['from_site'] = from_site
if item['from_site'] == '6pm' or item['from_site'] == '6pm/6pm':
item['from_site'] = 'sixpm'
spu_id = re.search('spuid=(.+)&?',response.url)
if spu_id:
spu_id = spu_id.group(1)
else:
spu_id = re.search('&spu=(.+)&?',response.url).group(1)
item['show_product_id'] = spu_id
item['url'] = goods_detail['pageUrl']
if self.editor_flag:
item['editor_flag'] = self.editor_flag
if not goods_detail['skuInfo']:
colorItem = Color()
images = []
color_names = []
skus=[]
for image in goods_detail['coverImgList']:
imageItem = ImageItem()
imageItem['image'] = image
imageItem['thumbnail'] = image
images.append(imageItem.copy())
colorItem['images'] = images
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
color_name = 'One Color'
if color_name not in color_names:
color_names.append(color_name)
colorItem['name'] = color_name
colorItem['cover'] = goods_detail['coverImgUrl']
yield colorItem
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = item['list_price']
skuItem['current_price'] = item['current_price']
skuItem['color'] = color_name
skuItem['id'] = item['show_product_id'] + 'onecolor'
skuItem['from_site'] = item['from_site']
if goods_detail['inStock'] == 0:
skuItem['is_outof_stock'] = True
skuItem['size'] = 'One Size'
skus.append(skuItem)
item['sizes'] = ['One Size']
else:
skus_info = goods_detail['skuInfo']['style']['skustylelist']
color_names = []
skus = []
sizes = []
dimensions_values = {}
for sku_info in skus_info:
colorItem = Color()
images = []
for image in sku_info['coverImgList']:
imageItem = ImageItem()
imageItem['image'] = image
imageItem['thumbnail'] = image
images.append(imageItem.copy())
colorItem['images'] = images
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
if sku_info['style']:
color_name = sku_info['style']
else:
color_name = 'One Color'
if color_name not in color_names:
color_names.append(color_name)
colorItem['name'] = color_name
colorItem['cover'] = images[0]['image']
yield colorItem
for sku in sku_info['data']:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = sku['mallPriceOrg']
skuItem['current_price'] =sku['realPriceOrg']
skuItem['color'] = color_name
skuItem['id'] = sku['skuid']
skuItem['from_site'] = item['from_site']
if sku['inStock'] == 0:
skuItem['is_outof_stock'] = True
if len(sku['attr']) == 0:
skuItem['size'] = 'One Size'
if skuItem['size'] not in sizes:
sizes.append(skuItem['size'])
else:
skuItem['size'] = {}
for attr in sku['attr']:
skuItem['size'][attr['attrName'].lower()] = attr['attrValue']
if attr['attrName'].lower() not in item['dimensions']:
item['dimensions'].append(attr['attrName'].lower())
if attr['attrName'].lower() not in dimensions_values.keys():
dimensions_values[attr['attrName'].lower()] = [attr['attrValue']]
else:
dimensions_values[attr['attrName'].lower()].append(attr['attrValue'])
if 'size' not in skuItem['size'].keys():
skuItem['size']['size'] = 'One Size'
dimensions_values['size'] = 'One Size'
skus.append(skuItem)
if sizes:
item['sizes'] = sizes
elif dimensions_values:
item['sizes'] = dimensions_values
else:
return
item['skus'] = skus
item['colors'] = color_names
yield item
def handle_parse_item(self, response, item):
pImgStr = "".join(re.findall(r'(pImgs[^;]+;)+', response.body))
context = execjs.compile('''
%s
function getPImgs(){
return pImgs;
}
''' % pImgStr)
pImgs = context.call('getPImgs')
sel = Selector(response)
outofstock_result = re.search(r'outOfStock[\s]*=[\s]*([^;]+);',
response.body)
if outofstock_result and outofstock_result.group(1) == 'true':
return
stock_json_result = re.search(r'var stockJSON[\s]*=[\s]*([^;]+);',
response.body)
if stock_json_result:
stock_dic = eval(stock_json_result.group(1))
if stock_dic:
color_price_dic = eval(
re.search(r'colorPrices[\s]*=[\s]*([^;]+);',
response.body).group(1))
style_id_dic = eval(
re.search(r'styleIds[\s]*=[\s]*([^;]+);',
response.body).group(1))
product_gender = eval(
re.search(r'productGender[\s]*=[\s]*([^;]+);',
response.body).group(1))
zeta_categories = eval(
re.search(r'zetaCategories[\s]*=[\s]*([^;]+);',
response.body).group(1))
category = eval(
re.search(r';[\s]*category[\s]*=[\s]*([^;]+);',
response.body).group(1))
sub_category = eval(
re.search(r'subCategory[\s]*=[\s]*("[^"]+"[\s]*);',
response.body).group(1))
dimension_dic = eval(
re.search(r'dimensions[\s]*=[\s]*([^;]+);',
response.body).group(1))
#dimToUnitToValJSON = eval(re.search(r'dimToUnitToValJSON[\s]*=[\s]*([^;]+);', response.body).group(1))
dimensionIdToNameJson = eval(
re.search(r'dimensionIdToNameJson[\s]*=[\s]*([^;]+);',
response.body).group(1))
valueIdToNameJSON = eval(
re.search(r'valueIdToNameJSON[\s]*=[\s]*([^;]+);',
response.body).group(1))
colorNames = eval(
re.search(r'colorNames[\s]*=[\s]*({[^}]+}[\s]*);',
response.body).group(1))
if len(zeta_categories) > 0:
item['product_type'] = zeta_categories[0].values()[0]
if category == item['product_type']:
item['category'] = sub_category
else:
item['category'] = category
item['sub_category'] = sub_category
else:
item['product_type'] = category
item['category'] = sub_category
if 'gender' in response.meta.keys():
meta_gender = response.meta['gender']
if product_gender.lower == 'unisex':
if meta_gender == 'boys' or meta_gender == 'girls':
item['gender'] = 'kid-unisex'
else:
item['gender'] = 'unisex'
else:
item['gender'] = meta_gender
'''跳过描述，过于复杂'''
size_info_images = []
desc = sel.xpath(
'//div[@id="productDescription"]//div[@itemprop="description"]/ul'
).extract()
if len(desc) > 0:
item['desc'] = desc[0]
size_infos = sel.xpath(
'//div[@id="productDescription"]//div[@itemprop="description"]/ul/li/a[@class="popup-570-550"]'
)
if len(size_infos) > 0:
size_info_images = []
for size_info in size_infos:
size_info_image_url = size_info.xpath(
'@href').extract()[0]
if not re.match(r'^http:\/\/', size_info_image_url):
size_info_image_url = self.base_url + size_info_image_url
size_info_images.append(size_info_image_url)
else:
desc_ul = sel.xpath(
'//div[@id="prdInfoText"]//span[@class="description summary"]/ul'
).extract()
if len(desc_ul) == 0:
return
item['desc'] = desc_ul[0]
size_infos = sel.xpath(
'//div[@id="prdInfoText"]//span[@class="description summary"]/ul/li/a[@class="popup-570-550"]'
)
if len(size_infos) > 0:
size_info_images = []
for size_info in size_infos:
size_info_image_url = size_info.xpath(
'@href').extract()[0]
if not re.match(r'^http:\/\/', size_info_image_url):
size_info_image_url = self.base_url + size_info_image_url
size_info_images.append(size_info_image_url)
if len(size_info_images) > 0:
item['size_info'] = {'images': size_info_images}
colors = []
'''处理color'''
for (color, color_name) in colorNames.items():
colorItem = Color()
colorItem['type'] = 'color'
colorItem['from_site'] = self.name
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = color_name
colors.append(color_name)
styleId = str(style_id_dic[color])
#colorItem['cover'] = sel.xpath('//a[@id="frontrow-'+color+'"]/img/@src').extract()[0]
if 'p' in pImgs[styleId]['DETAILED'].keys():
colorItem['cover'] = pImgs[styleId]['DETAILED']['p']
elif 'd' in pImgs[styleId]['DETAILED'].keys():
colorItem['cover'] = pImgs[styleId]['DETAILED']['d']
elif '1' in pImgs[styleId]['MULTIVIEW_THUMBNAILS'].keys():
colorItem['cover'] = pImgs[styleId][
'MULTIVIEW_THUMBNAILS']['1']
elif '4' in pImgs[styleId]['MULTIVIEW_THUMBNAILS'].keys():
colorItem['cover'] = pImgs[styleId][
'MULTIVIEW_THUMBNAILS']['4']
elif '5' in pImgs[styleId]['MULTIVIEW_THUMBNAILS'].keys():
colorItem['cover'] = pImgs[styleId][
'MULTIVIEW_THUMBNAILS']['5']
colorImages = pImgs[styleId]
thumbImages = colorImages['MULTIVIEW_THUMBNAILS']
images = colorImages['2x']
if len(images) == 0:
images = colorImages['MULTIVIEW']
thumbImages = sorted(thumbImages.iteritems(),
key=lambda d: d[0])
images_array = []
for image_tuple in thumbImages:
imageItem = ImageItem()
if image_tuple[0] in images.keys():
imageItem['image'] = images[image_tuple[0]]
imageItem['thumbnail'] = image_tuple[1]
if image_tuple[0] == 'p' or image_tuple[0] == 'd':
images_array.insert(0, imageItem)
else:
images_array.append(imageItem)
colorItem['images'] = images_array
yield colorItem
item['colors'] = colors
dimensions = []
sizes = {}
for dimension in dimension_dic:
dimensions.append(dimensionIdToNameJson[dimension])
sizes[dimensionIdToNameJson[dimension]] = []
if len(dimensions) == 0:
dimensions = ['size']
if len(sizes) == 0:
sizes = {'size': ['onesize']}
item['dimensions'] = dimensions
'''处理sku库存'''
skuCollectionsList = []
for sku_stock in stock_dic:
color = sku_stock['color']
if color in color_price_dic.keys():
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['from_site'] = self.name
skuItem['show_product_id'] = item['show_product_id']
skuItem['id'] = sku_stock['id']
skuItem["list_price"] = color_price_dic[color]['wasInt']
skuItem['current_price'] = color_price_dic[color]['nowInt']
skuItem['color'] = colorNames[color]
size_demension = {}
for demension in dimension_dic:
if demension in sku_stock.keys(
) and sku_stock[demension] in valueIdToNameJSON.keys():
size_value = valueIdToNameJSON[
sku_stock[demension]]['value']
size_demension[
dimensionIdToNameJson[demension]] = size_value
if not size_value in sizes[
dimensionIdToNameJson[demension]]:
sizes[dimensionIdToNameJson[demension]].append(
size_value)
if len(size_demension) == 0:
size_demension = {'size': 'onesize'}
skuItem['size'] = size_demension
skuItem['quantity'] = sku_stock['onHand']
skuItem['is_outof_stock'] = False
skuCollectionsList.append(skuItem)
item['skus'] = skuCollectionsList
item['sizes'] = sizes
item = self.handle_dimension_to_name(response, item,
dimensionIdToNameJson)
yield item
def handle_parse_item(self, response, item):
sel = Selector(response)
if len(sel.xpath('//form[@id="product-form"]//meta').extract()) > 1:
return
if len(sel.xpath('//div[@class="sold-out-details"]')) > 0:
return
item['show_product_id'] = sel.xpath(
'//div[@class="product-code"]/span/text()').extract()[0].strip()
imgs = sel.xpath(
'//div[@class="container-imagery"]//ul[@class="thumbnails no-carousel"]/li/img/@src'
).extract()
if len(imgs) == 0:
imgs = sel.xpath(
'//div[@class="container-imagery"]//ul[@class="swiper-wrapper"]/li/img/@src'
).extract()
images = []
for img in imgs:
if 'http:' not in img:
img = 'http:' + img
if 'xs.jpg' in img:
img = img.replace('xs.jpg', 'pp.jpg')
imageItem = ImageItem()
imageItem['image'] = img
imageItem['thumbnail'] = img.replace('pp.jpg', 'm.jpg')
images.append(imageItem)
colorItem = Color()
colorItem['images'] = images
colorItem['type'] = 'color'
colorItem['from_site'] = item['from_site']
colorItem['show_product_id'] = item['show_product_id']
colorItem['name'] = 'One Color'
colorItem['cover'] = images[0]['image'].replace('pp.jpg', 'xs.jpg')
# colorItem['cover'] = images[0]['image'].split('_')[0] + '_sw.jpg'
# print colorItem['cover']
# req = requests.get(colorItem['cover'])
# if not req.ok:
# colorItem['cover'] = images[0]['image'].replace('pp.jpg', 'xs.jpg')
yield colorItem
price = int(
sel.xpath('//form[@id="product-form"]/meta/@data-price-full').
extract()[0]) / 100
if len(sel.xpath('//select-dropdown[@class="sku"]/@options')) > 0:
sku_str = sel.xpath(
'//select-dropdown[@class="sku"]/@options').extract()[0]
skus = json.loads(sku_str)
item['skus'] = []
sizes = []
for sku in skus:
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = price
skuItem['current_price'] = price
skuItem['color'] = 'One Color'
skuItem['size'] = sku['data']['size']
sizes.append(sku['data']['size'])
skuItem['id'] = sku['id']
skuItem['from_site'] = item['from_site']
if sku['stockLevel'] == 'In_Stock' or sku[
'stockLevel'] == 'Low_Stock':
skuItem['is_outof_stock'] = False
else:
skuItem['is_outof_stock'] = True
item['skus'].append(skuItem)
else:
item['skus'] = []
skuItem = SkuItem()
skuItem['type'] = 'sku'
skuItem['show_product_id'] = item['show_product_id']
skuItem['list_price'] = price
skuItem['current_price'] = price
skuItem['color'] = 'One Color'
sizes = ['One Size']
skuItem['size'] = 'One Size'
skuItem['id'] = sel.xpath(
'//input [@class="sku"]/@value').extract()[0]
stock_level = sel.xpath(
'//input [@class="sku"]/@data-stock').extract()[0]
if stock_level == 'In_Stock' or stock_level == 'Low_Stock':
skuItem['is_outof_stock'] = False
else:
skuItem['is_outof_stock'] = True
skuItem['from_site'] = item['from_site']
item['skus'].append(skuItem)
item['gender'] = self.gender
item['colors'] = ['One Color']
item['sizes'] = sizes
item['desc'] = ''
if len(sel.xpath('//widget-show-hide[@id="accordion-1"]//ul/li')) > 0:
item['desc'] = item['desc'] + sel.xpath(
'//widget-show-hide[@id="accordion-1"]//ul/li').extract()[0]
if len(sel.xpath('//widget-show-hide[@id="accordion-2"]//ul/li')) > 0:
item['desc'] = item['desc'] + sel.xpath(
'//widget-show-hide[@id="accordion-2"]//ul/li').extract()[0]
if len(sel.xpath('//widget-show-hide[@id="accordion-2"]//p')) > 0:
item['desc'] = item['desc'] + sel.xpath(
'//widget-show-hide[@id="accordion-2"]//p').extract()[0]
product_items = sel.xpath(
'//widget-show-hide[@name="Editor\'s Notes"]/div[@class="show-hide-content"]/div/p/a'
)
if len(product_items) > 0:
related_items_id = []
for product_item in product_items:
product_id = product_item.xpath('./@href').extract()[0].split(
'/')[-1]
related_items_id.append(product_id)
if related_items_id:
item['related_items_id'] = related_items_id
media_url = 'https://video.net-a-porter.com/videos/productPage/' + item[
'show_product_id'] + '_detail.mp4'
try:
req = requests.head(media_url)
if req.ok:
item['media_url'] = media_url
except Exception as e:
logging.error('error media url: ' + media_url + ' error msg: ' +
str(e))
yield item