def get_perksoffer_weekly_sale_in_detail(self, response):
name_key1s = ['//div', '//h4', '/text()']
name_key2s = ['product-title qa_modelName', None, None]
name_pattern = tp.fast_gen_pattern(name_key1s, name_key2s)
names = response.xpath(name_pattern).extract()
pricing_summary_key1s = ['//div', '//dl']
pricing_summary_key2s = ['pricing', 'pricingSummary-details']
pricing_summary_pattern = tp.fast_gen_pattern(pricing_summary_key1s,
pricing_summary_key2s)
pricing_summarys = response.xpath(pricing_summary_pattern)
if len(names) != len(pricing_summarys):
raise ValueError(
'logical error: pricingSummarys do not relate with model names'
)
web_price_key1s = ['.//div', '//div', '//dd', '/text()']
web_price_key2s = [
'price-block', 'webprice qa_webPrice', 'webprice-final', None
]
web_price_pattern = tp.fast_gen_pattern(web_price_key1s,
web_price_key2s)
saving_price_key1s = ['.//div', '//div', '//span', '/text()']
saving_price_key2s = [
'price-block', 'yousave-section qa_savings',
'yousave qa_savingsPrice', None
]
saving_price_pattern = tp.fast_gen_pattern(saving_price_key1s,
saving_price_key2s)
products = []
for name, pricing_summary in zip(names, pricing_summarys):
name = tp.check_name(name, pass_filters, reject_filters, False)
if name is None:
continue
web_price = pricing_summary.xpath(web_price_pattern).extract()
saving_price = pricing_summary.xpath(
saving_price_pattern).extract()
web_price = tp.check_price('web price', web_price)
saving_price = tp.check_price('saving price', saving_price)
discount = 0
if web_price > 0.0:
discount = int(round((saving_price / web_price) * 100))
product = PerksOfferWeeklySaleItem()
product['end_date'] = self.end_date
product['model'] = name
product['save_up'] = str(discount) + '%'
products.append(product)
# from scrapy.shell import inspect_response
# inspect_response(response, self)
return products
def get_perksoffer_consumer_sale_in_detail(self, response):
products_k1s = ['//div', '//ul', '//li']
products_k2s = [
'weeklyDealsContainer', 'non-featured-products',
'product-column v[filters]'
]
products_k3s = ['id', None, None]
products_pattern = tp.fast_gen_pattern(products_k1s, products_k2s,
products_k3s)
non_featured_products = response.xpath(products_pattern)
name_k1s = ['.//div', '//h4', '/text()']
name_k2s = ['product-title qa_modelName', None, None]
name_pattern = tp.fast_gen_pattern(name_k1s, name_k2s)
web_price_k1s = ['.//div', '//div', '//div', '//dd', '/text()']
web_price_k2s = [
'pricing', 'price-block', 'webprice qa_webPrice', 'webprice-final',
None
]
web_price_pattern = tp.fast_gen_pattern(web_price_k1s, web_price_k2s)
saving_price_k1s = ['.//div', '//div', '//div', '//span', '/text()']
saving_price_k2s = [
'pricing', 'price-block', 'yousave-section qa_savings',
'yousave qa_savingsPrice', None
]
saving_price_pattern = tp.fast_gen_pattern(saving_price_k1s,
saving_price_k2s)
products = []
for non_featured_product in non_featured_products:
name = non_featured_product.xpath(name_pattern).extract()
name = tp.check_name(name, pass_filters, None, False)
if name is None:
continue
web_price = non_featured_product.xpath(web_price_pattern).extract()
saving_price = non_featured_product.xpath(
saving_price_pattern).extract()
web_price = tp.check_price('web price', web_price)
saving_price = tp.check_price('saving price', saving_price)
discount = 0
if web_price > 0.0:
discount = int(round((saving_price / web_price) * 100))
product = PerksOfferConsumerSaleItem()
product['end_date'] = self.end_date
product['model'] = name
product['save_up'] = str(discount) + '%'
products.append(product)
return products
def get_perksoffer_specified_product(self, response):
price_keys = [
(
'//div',
None,
'mainContent contentContainer pageWrapper ',
),
(
'//div',
'id',
'longscroll-subseries',
),
(
'//div',
None,
'hero-column hero-column-one',
),
(
'//div',
None,
'cta',
),
(
'//dl',
None,
'cta-price',
),
(
'//dd',
'itemprop',
'price',
),
('/text()', None, None),
]
price_pattern = tp.fast_gen_pattern2(price_keys)
price = response.xpath(price_pattern).extract()
price = tp.check_price('starting price', price)
# from scrapy.shell import inspect_response
# inspect_response(response, self)
product = PerksOfferProductItem()
product['model'] = self.product_name
product['starting_price'] = price
return product
def get_outlet_specified_products(self, response):
results_keys = [
('//div', None, 'mainContent contentContainer pageWrapper '),
('//div', 'id', 'results-area'),
('//div', 'id', 'resultsList'),
('//div', None,
'facetedResults-item only-allow-small-pricingSummary'),
]
results_pattern = tp.fast_gen_pattern2(results_keys)
results = response.xpath(results_pattern)
name_keys = [
('.//div', None, 'facetedResults-header'),
('//h3', None, 'facetedResults-title'),
('//a', None, 'facetedResults-cta'),
('/text()', None, None),
]
name_pattern = tp.fast_gen_pattern2(name_keys)
price_keys = [('.//div', None, 'facetedResults-body'),
('//div', None, 'pricingSummary'),
('//dd', None, 'pricingSummary-details-final-price'),
('/text()', None, None)]
price_pattern = tp.fast_gen_pattern2(price_keys)
status_keys = [
('.//div', None, 'facetedResults-body'),
('//div', None, 'pricingSummary'),
('//div', None, 'pricingSummary-secondary-details'),
('//span', None, 'rci-msg'),
('/text()', None, None),
]
status_pattern = tp.fast_gen_pattern2(status_keys)
configurations_keys = [
('.//div', None, 'facetedResults-body'),
('//div', None, 'facetedResults-feature-list'),
('//dd', None, None),
('/text()', None, None),
]
configurations_pattern = tp.fast_gen_pattern2(configurations_keys)
products = []
for item in results:
try:
name = item.xpath(name_pattern).extract()[0]
status = item.xpath(status_pattern).extract()[0]
price = item.xpath(price_pattern).extract()[0]
price = tp.check_price('price', price)
except IndexError as e:
self.logger.error('!!! invalid product detail: ' +
item.extract())
continue
name = tp.check_name(name, pass_filters, reject_filters, False)
if name is None:
continue
if not tp.price_is_within_limits(price):
continue
configurations = item.xpath(configurations_pattern).extract()
configuration = ''
for cfg in configurations:
cfg = tp.strip_invisible_chars(cfg)
if not tp.is_in_filters(cfg_pass_filters, cfg, False):
continue
if len(cfg) > 0:
configuration += cfg + '\t'
configuration = configuration[:-1]
product = OutletProductItem()
product['model'] = name
product['price'] = price
product['status'] = tp.strip_invisible_chars(status)
product['configuration'] = configuration
products.append(product)
return products