def parse_product(self, response):
meta = response.meta.copy()
product = meta.get('product', ProductItem())
if 'Good thing this is not permanent' in response.body_as_unicode():
# product['not_found'] = True
return
maintenance_error = response.xpath(
'.//*[contains(text(), "The site is currently under maintenance.")]'
)
if maintenance_error:
self.log(
"Website under maintenance error, retrying request: {}".format(
response.url), WARNING)
return Request(response.url,
callback=self.parse_product,
meta=response.meta,
dont_filter=True)
if response.status == 429:
response = requests.get(url=response.url, timeout=5)
# Parse name
name = self._parse_name(response)
product['name'] = name
# Parse image
image = self._parse_image(response)
product['image'] = image
# Parse model
model = self._parse_model(response)
product['model'] = model
# Parse upc
# upc = self._parse_upc(response)
# product['upc'] = upc
# Parse currencycode
product['currencycode'] = 'USD'
# Set locale
product['locale'] = 'en-US'
# Parse sku
sku = self._parse_sku(response)
product['sku'] = sku
# Parse manufacturer
# manufacturer = self._parse_manufacturer(response)
# product['manufacturer'] = manufacturer
# Parse categories
categories = self._parse_categories(response)
product['categories'] = categories
# Parse retailer_key
retailer_key = self._parse_retailer_key(response)
product['retailer_key'] = retailer_key
# Parse in_store
in_store = self._parse_instore(response)
product['instore'] = in_store
# Parse stock status
response.meta['product'] = product
oos = self._parse_product_stock_status(response)
cond_set_value(product, 'productstockstatus', oos)
# Parse ship to store
# ship_to_store = self._parse_shiptostore(response)
# product['shiptostore'] = ship_to_store
# Parse gallery
product['gallery'] = self._parse_gallery(response)
# Parse features
# features = self._parse_features(response)
# product['features'] = features
# Parse condition
product['condition'] = 1
# Parse price
price = self._parse_price(response)
product['price'] = price
return product
def parse_product(self, response):
meta = response.meta
product = meta.get('product', ProductItem())
# Parse locate
locale = 'en_US'
cond_set_value(product, 'locale', locale)
# Parse name
name = self.parse_name(response)
cond_set(product, 'name', name, conv=string.strip)
# Parse image
image = self.parse_image(response)
cond_set(product, 'image', image)
# Parse brand
brand = self.parse_brand(response)
cond_set_value(product, 'brand', brand)
# Parse sku
sku = self.parse_sku(response)
cond_set_value(product, 'sku', sku)
# Parse price
price = self.parse_price(response)
cond_set_value(product, 'price', price)
# Parse sale price
product['saleprice'] = price
# Parse model
model = self._parse_model(response)
cond_set_value(product, 'model', model)
# Parse gallery
gallery = self._parse_gallery(response)
product['gallery'] = gallery
# Parse stock status
oos = self._parse_product_stock_status(response)
cond_set_value(product, 'productstockstatus', oos)
# Parse categories
categories = self._parse_categories(response)
cond_set_value(product, 'categories', categories)
# Parse manufacturer
manufacturer = self._parse_manufacturer(response)
cond_set_value(product,
'manufacturer',
manufacturer,
conv=string.strip)
# Parse shipping phrase
shipping_phrase = self._parse_shippingphrase(response)
product['shippingphrase'] = shipping_phrase
# Parse ship to store
ship_to_store = self._parse_shiptostore(response)
product['shiptostore'] = ship_to_store
# Parse retailer_key
retailer_key = self._parse_retailer_key(response)
product['retailer_key'] = retailer_key
# Parse features
features = self._parse_features(response)
product['features'] = features
return product
def parse_product(self, response):
meta = response.meta
product = meta.get('product', ProductItem())
# Parse locate
locale = 'en_US'
cond_set_value(product, 'locale', locale)
# Parse name
name = self.parse_name(response)
cond_set(product, 'name', name, conv=string.strip)
# Parse image
image = self.parse_image(response)
cond_set(product, 'image', image)
# Parse brand
brand = self.parse_brand(response)
cond_set_value(product, 'brand', brand)
# Parse sku
sku = self.parse_sku(response)
cond_set_value(product, 'sku', sku)
# Parse price
price = self.parse_price(response)
cond_set_value(product, 'price', price)
# Parse sale price
product['saleprice'] = price
# Parse model
model = self._parse_model(response)
cond_set_value(product, 'model', model)
# Parse gallery
gallery = self._parse_gallery(response)
product['gallery'] = gallery
# Parse stock status
oos = self._parse_product_stock_status(response)
cond_set_value(product, 'productstockstatus', oos)
# Parse categories
categories = self._parse_categories(response)
cond_set_value(product, 'categories', categories)
# Parse manufacturer
manufacturer = self._parse_manufacturer(response)
cond_set_value(product, 'manufacturer', manufacturer, conv=string.strip)
# Parse shipping phrase
shipping_phrase = self._parse_shippingphrase(response)
product['shippingphrase'] = shipping_phrase
# Parse ship to store
ship_to_store = self._parse_shiptostore(response)
product['shiptostore'] = ship_to_store
# Parse retailer_key
retailer_key = self._parse_retailer_key(response)
product['retailer_key'] = retailer_key
# Parse features
features = self._parse_features(response)
product['features'] = features
return product