def _create_product_dictionary(
self,
response: HtmlResponse,
data: Optional[Dict] = None,
) -> product.Product:
try:
upc = (universal_product_code.UniversalProductCode(
upc=data.get('ProductId').replace('_', ''))).value
except:
# TODO: Log issue and return nothing.
return None
title1 = response.css('meta[property="og:title"]::attr(content)'
).extract()[0].split('|')[0]
title2 = response.css('title::text').get()
name = title1 or title2
if not name:
pass # TODO: Log error and return none.
elif name == 'Grocery Product' or name == 'Produit épicerie en ligne':
pass # TODO: Log error and return none.
brand = data.get('BrandName')
if not name:
pass # TODO: Log error and return none.
item_loader = product_item_loader.ProductItemLoader(
response=response
).add_name(
response=response,
name=name, # TODO: What about if it's none.
language=self.language,
).add_brand(
response=response,
brand=brand, # TODO: What about if it's none.
language=self.language,
).add_upc(response=response, upc=upc) \
.add_product_data_dictionary(
product_data_dictionary=self._create_product_data_dictionary(
response=response,
data=data,
name=name,
brand=brand,
upc=upc,
),
).add_offer_dictionary(
offer_dictionary=self._create_offer_dictionary(
response=response,
data=data,
),
).add_store_dictionary(
store_dictionary=self._create_store_dictionary(
response=response,
),
).add_supported_language(language=self.language)
return item_loader.load_item()
def _create_product_dictionary(
self,
response: HtmlResponse,
data: Optional[Dict] = None,
) -> product.Product:
try:
upc = (universal_product_code.UniversalProductCode(
upc=response.css('span[itemprop="sku"]::text').get())).value
except Exception as exception:
logging.exception(msg='Unable to get UPC.', exc_info=exception)
return None
name1 = response.css(
"div.product-info.item-addToCart > a.invisible-text::text"
).extract()
name2 = response.css('title::text').extract()[0].split('|')[0]
name = name1 or name2
if not name:
pass # TODO: Log error and return none.
brand = response.css('div[itemtype="http://schema.org/Product"] \
> span[itemprop="brand"]::text').extract()
item_loader = product_item_loader.ProductItemLoader(
response=response
).add_name(
response=response,
name=name,
language=self.language,
).add_brand(
response=response,
brand=brand,
language=self.language,
).add_upc(response=response, upc=upc) \
.add_product_data_dictionary(
product_data_dictionary=self._create_product_data_dictionary(
response=response,
data=data,
name=name,
brand=brand,
upc=upc,
),
).add_offer_dictionary(
offer_dictionary=self._create_offer_dictionary(
response=response,
data=data,
),
).add_store_dictionary(
store_dictionary=self._create_store_dictionary(
response=response,
),
).add_supported_language(language=self.language)
return item_loader.load_item()
def parse_product(self, response):
productLoader = product_item_loader.ProductItemLoader(
response=response)
productLoader.add_css('name', ['title'])
productLoader.add_css(
'description', ['head > meta[name="description"]::attr(content)'])
productLoader.add_value('currentPrice', [self.__get_price(response)])
productLoader.add_value('url', [response.url])
productLoader.add_css('availability',
['div.availability-text::attr(content)'])
productLoader.add_css('tags',
['head > meta[name="keywords"]::attr(content)'])
productLoader.add_css(
'upc', ['div.product-details-numbers-wrapper > div:nth-child(2)'])
return productLoader.load_item()
def parse_product(self, response):
productLoader = product_item_loader.ProductItemLoader(
response=response)
productLoader.add_css(
'name', ['head > meta[property="og:title"]::attr(content)'])
productLoader.add_css(
'description', ['head > meta[name="description"]::attr(content)'])
productLoader.add_css('releaseDate',
['#ctl00_CP_ctl00_PD_lblReleaseDate'])
productLoader.add_value('currentPrice', [self.__get_price(response)])
productLoader.add_value('url', [response.url])
productLoader.add_css('availability', [
'#schemaorg-offer > div.price-module.clearfix > div.price-wrapper.price-extra-large > link[itemprop="availability"]::attr(href)'
])
productLoader.add_css('tags',
['head > meta[name="keywords"]::attr(content)'])
return productLoader.load_item()
def __load_with_dictionary(self, response, data):
product_loader = product_item_loader.ProductItemLoader(
response=response)
data['tags'] = self.__parse_tags(data['tags'])
try:
upc = (universal_product_code.UniversalProductCode(
data['tags']['upc'])).value
except:
upc = None
if upc:
product_loader.add_value(
product.Product.KEY_GTIN,
super()._create_gtin_field(response=response,
type=global_trade_item_number.
GlobalTradeItemNumber.UPCA.value,
value=upc))
product_loader.add_value(field_name=product.Product.KEY_BRAND,
value=data['vendor'])
product_loader.add_value(field_name=product.Product.KEY_CURRENT_OFFER,
value=self.__create_offer_dictionary(
response, data))
product_loader.add_value(field_name=product.Product.KEY_MODEL_NUMBER,
value=data.get('tags').get('vsn')
or data.get('variants')[0].get('sku'))
product_loader.add_value(field_name=product.Product.KEY_PRODUCT_DATA,
value=self._create_product_data_dictionary(
response, data, upc))
product_loader.add_value(
field_name=product.Product.KEY_STORE,
value=self.__create_store_dictionary(response))
return product_loader.load_item()