def parse_product_page(self, response): now = datetime.datetime.now() date = now.strftime("%Y-%m-%d") for product in self.products: if product.lower() in response.request.url: product_match = product normal_div = '//div[@class="priceAndAvail"]//div[@class="productPrice nobefore"]//strong/text()' discount_div = '//div[@class="priceAndAvail"]//div[@class="productPrice"]//strong/text()' normal_container = response.xpath(normal_div) discount_container = response.xpath(discount_div) if len(discount_container) > 0: price_div = discount_div else: price_div = normal_div # print('============================') # print(normal_container) # print(discount_container) # print('============================') p = ProductLoader(item=Product(), response=response) p.add_value('product', product_match) p.add_xpath('price', price_div) p.add_value('date', date) p.add_value('retailer', "Bilka") yield p.load_item() w = WebsiteLoader(item=Website(), response=response) w.add_value('html', response.body) w.add_value('date', date) w.add_value('product', product_match) w.add_value('retailer', "Bilka") yield w.load_item()
def parse_product_page(self, response): now = datetime.datetime.now() date = now.strftime("%Y-%m-%d") for product in self.products: if product.lower() in response.request.url: product_match = product p = ProductLoader(item=Product(), response=response) p.add_value('product', product_match) normal_div = '//div[@class="vip__price-box-price"]/text()' discount_div = '//div[@class="campaign-banner__info__right"]//span[@class="top"]/text()' normal_container = response.xpath(normal_div) discount_container = response.xpath(discount_div) if len(discount_container) > 0: price_div = discount_div else: price_div = normal_div print('============================') print(normal_container) print(discount_container) print('============================') p.add_xpath('price', price_div) p.add_value('date', date) p.add_value('retailer', "Skousen") yield p.load_item() w = WebsiteLoader(item=Website(), response=response) w.add_value('html', response.body) w.add_value('date', date) w.add_value('product', product_match) w.add_value('retailer', "Skousen") yield w.load_item()
def parse_product_page(self, response): now = datetime.datetime.now() date = now.strftime("%Y-%m-%d") for product in self.products: if product.lower() in response.request.url: product_match = product p = ProductLoader(item=Product(), response=response) p.add_value('product', product_match) normal_div = '//span[@class="product-price-tag"]/text()' discount_div = '//span[@class="product-price-tag"]/text()' normal_container = response.xpath(normal_div) discount_container = response.xpath(discount_div) if len(discount_container) > 0: price_div = discount_div else: price_div = normal_div p.add_xpath('price', price_div) p.add_value('date', date) p.add_value('retailer', "Punkt1") yield p.load_item() w = WebsiteLoader(item=Website(), response=response) w.add_value('html', response.body) w.add_value('date', date) w.add_value('product', product_match) w.add_value('retailer', "Punkt1") yield w.load_item()
def parse(self, response): now = datetime.datetime.now() date = now.strftime("%Y-%m-%d") product_match = response.request.url.split('/')[-2] p = ProductLoader(item=Product(), response=response) p.add_value('product', product_match) p.add_xpath('price', '//div[@class="product-price-container"]//span/text()') p.add_value('date', date) p.add_value('retailer', "Elgiganten") yield p.load_item() w = WebsiteLoader(item=Website(), response=response) w.add_value('html', response.body) w.add_value('date', date) w.add_value('product', product_match) w.add_value('retailer', "Elgiganten") yield w.load_item()
def parse(self, response): now = datetime.datetime.now() date = now.strftime("%Y-%m-%d") p = ProductLoader(item=Product(), response=response) p.add_xpath('price', '//div[@class="prices-container "]//span/text()') p.add_value('date', date) p.add_value('retailer', "Power") product_name = response.xpath( '//div[@class="container pad"]//h1/text()').extract_first() for product in self.products: if product.lower() in product_name.lower(): product_match = product p.add_value('product', product_match) yield p.load_item() w = WebsiteLoader(item=Website(), response=response) w.add_value('html', response.body) w.add_value('date', date) w.add_value('product', product_match) w.add_value('retailer', "Power") yield w.load_item()
def parse_product_page(self, response): now = datetime.datetime.now() date = now.strftime("%Y-%m-%d") product_match = 'unkown' for product in self.products: if product.lower() in response.request.url: product_match = product p = ProductLoader(item=Product(), response=response) p.add_value('product', product_match) p.add_xpath('price', '//div[@class="vip__price-box-price"]/text()') p.add_value('date', date) p.add_value('retailer', "Whiteaway") yield p.load_item() w = WebsiteLoader(item=Website(), response=response) w.add_value('html', response.body) w.add_value('date', date) w.add_value('product', product_match) w.add_value('retailer', "Whiteaway") yield w.load_item()