def single_result(self, response):
""" single result parser
Parser that will be used if response result detected as single item page
Args:
response
Returns:
item
"""
item = ElectronicItem()
items = []
part_number = cleansplit(
Selector(text=response.body).css(
'dd[itemprop="http://schema.org/sku"]::text').extract_first())
manufacturer_part_number = cleansplit(
Selector(text=response.body).css(
'dd[itemprop="mpn"]::text').extract_first())
manufacturer_name = cleansplit(
Selector(text=response.body).css(
'span[itemprop="http://schema.org/manufacturer"]::text').
extract_first())
description = cleansplit(
Selector(text=response.body).css(
'div[itemprop="http://schema.org/description"]::text').
extract_first())
'''Quantity stock available always 29,050,000 for yes.'''
quantity_available = cleansplit(
Selector(text=response.body).xpath(
"//span[@class='availability']//text()").extract_first())
# .xpath('//ul[@class="BuyingOptions-labeledValues BuyingOptions-labeledValues--right"]'
# '//li[1]//strong//text()').extract_first())
image_url = cleansplit(
Selector(text=response.body)
#.xpath("/img[@id='productMainImage']/@src").extract_first())
.css("img[id='productMainImage']::attr(src)").extract_first())
item['manufacturer'] = manufacturer_name
item['manufacturer_part_number'] = manufacturer_part_number
item['supplier'] = self.spider_name
item['supplier_part_number'] = part_number
item['description'] = description
item['stock_qty'] = cleanqty(quantity_available)
item['product_url'] = response.url
item['image_url'] = image_url
yield item
def parse_search_result(self, response):
""" Parse Search Result
Parser that will be used if response result detected as search result page.
Args:
response
Returns:
item
"""
item = ElectronicItem()
part_number = cleansplit(
Selector(text=response.body)
#.css("li.ttipartnumber a ::text/li[@class='ttipartnumber']/a/text()"))
.css("p.sku a::text"))
manufacturer_part_number = cleansplit(
Selector(text=response.body)
#.xpath("/li[@class='mfrpartnumber']/a/text()"))
.css("td.productImage.mftrPart a::text"))
manufacturer_name = cleansplit(
Selector(text=response.body)
#.xpath("//id('sProdList')/x:tbody/x:tr/x:td/x:a/x:p[1]::text"))
.css("td.description a p:first-of-type::text"))
description = cleansplit(
Selector(text=response.body)
#.xpath("/td[@class='description']/text()"))
.css("td.description a p:nth-of-type(2)::text"))
quantity_available = cleansplit(
Selector(text=response.body)
#.xpath("/td[@class='availability']/text()"))
.css("span.inStockBold::text"))
image_url = cleansplit(
Selector(text=response.body).xpath(
"//img[@class='productThumbnail']").xpath('@src'))
'''
This is variable handler when no content in selected xpath. so this algorithm will keep list balanced.
and alyways will process zip iteration. and return scaped item. see customfunction.py for listbalancer method'''
if not quantity_available:
quantity_available = listbalancer(part_number)
if not image_url: image_url = listbalancer(image_url)
if not description: description = listbalancer(description)
for i, j, k, l, m, n in zip(part_number, manufacturer_part_number,
manufacturer_name, description,
quantity_available, image_url):
item['manufacturer'] = k
item['manufacturer_part_number'] = j
item['supplier'] = self.spider_name
item['supplier_part_number'] = i
item['description'] = l
item['image_url'] = n
item['product_url'] = response.url
item['stock_qty'] = cleanqty(m.replace(u'\xa0', u''))
yield item
next_url = response.xpath(
"//span[@class='current']"
"/following-sibling::span/a/@href").extract_first()
if self.debug: print "Next URL -> %s" % (next_url)
if next_url:
"Following Next Page {0}".format(response.urljoin(next_url))
yield Request(response.urljoin(next_url),
callback=self.parse_search_result,
dont_filter=True)