def parse_product_list_page(self, html_page):
        """Parse the HTML page which has the product list and populate the product_dict"""

        html_tree = lxml.html.fromstring(html_page, parser=lxml.html.HTMLParser(encoding='utf-8'))

        product_list = html_tree.xpath("//table[@class='tablaproductos']/tbody/tr")
        #print len(product_list)

        for product_item in product_list:
            # Check whether the product is available
            if len(product_item.xpath("./td[1]/img[@alt='PRODUCTOS NO DISPONIBLES']")) > 0:
                continue

            product_id = product_item.xpath("./td[4]/input/@value")[0].partition(';')[0]

            product_name = product_item.xpath("./td[1]//label")[0].text.replace(' ***LE RECOMENDAMOS***', '')
            product_name = product_name.encode('utf-8')

            product_price = float(product_item.xpath("./td[2]/span")[0].text.partition(' ')[0].replace(',', '.'))

            product_unitary_price = product_item.xpath("./td[2]/span[contains("
                                                       "concat(' ', normalize-space(@class), ' '), ' precio_ud ')]")
            if len(product_unitary_price) > 0:
                product_unitary_price = product_unitary_price[0].text.partition(': ')
                product_unit = product_unitary_price[0]
                product_unitary_price = float(product_unitary_price[2].partition(' ')[0].replace(',', '.'))

                # The following are fixings to normalize shop "bugs"
                if product_id == '43401':
                    product_unitary_price = product_price
                    product_unit = '1 UNIDAD'
                elif product_id == '40805':
                    amount = re.search(r'(\d+) LAVADOS', product_name).group(1)
                    product_unitary_price = round(round(product_price / float(amount), 4), 2)
                    product_unit = '1 LAVADO'

                product_unitary_price, product_unit = self.normalize_unitary_price(product_unitary_price, product_unit)
            else:
                product_unitary_price, product_unit = self.get_unitary_price(product_price, product_name)

            Shop.add_product(self,
                             Product(product_id, product_name, product_price, product_unitary_price, product_unit))
    def parse_product_list_page(self, html_page):
        """Parse the HTML page which has the product list and populate the product_dict"""

        html_tree = lxml.html.fromstring(html_page, parser=lxml.html.HTMLParser(encoding='utf-8'))

        product_list = html_tree.xpath("//table[@id='conte']/form/tr[starts-with(@id, 'prod_') or "
                                       "starts-with(@id, 'categ_')]")
        #print len(product_list)

        for product_item in product_list:
            # Get product category
            product_id = product_item.attrib['id']
            if product_id.startswith('categ_'):
                if not product_id.endswith('_2'):
                    product_category = product_item.xpath("./td/table/tr/td[2]/p")[0].text.strip(' >')
                    #print product_category
                continue

            base_xpath = "./td/table/tr/td/table/tr/td[3]/table"
            # Check whether the product is available
            if len(product_item.xpath(base_xpath + "/tr[3]/td/table/tr/td[@class='sub_menu_11']/a/"
                                                   "strong[text()='Busca Sustituto']")) > 0:
                continue

            product_id = product_id.partition('_')[2]
            product_name = product_item.xpath(base_xpath + "/tr/td/table/tr/td[@class='menu_sup11']")[0].\
                                              text_content().strip()
            product_name = product_name.encode('utf-8')
            product_price = float(product_item.xpath(base_xpath +
                                                     "/tr[3]/td/table/tr/td[@class='menu_12_rojo_sin']/strong")[0].\
                                                     text.partition(' ')[0].replace(',', '.'))

            # The following are fixings to normalize shop "bugs"
            if product_id == '900782_2058535':
                product_name = product_name.replace('3x80', '3x60')

            product_unitary_price, product_unit = self.get_unitary_price(product_price, product_name, product_category)

            Shop.add_product(self,
                             Product(product_id, product_name, product_price, product_unitary_price, product_unit))
    def parse_product_list_page(self, html_page):
        """Parse the HTML page which has the product list and populate the product_dict"""

        html_tree = lxml.html.fromstring(html_page, parser=lxml.html.HTMLParser(encoding='utf-8'))

        product_list = html_tree.xpath("//table[@id='shopping-cart-table']/tbody/tr[not(@class)]")
        #print len(product_list)

        for product_item in product_list:
            # Check whether the product is available
            if len(product_item.xpath("./td[3]/span[text()='Producto no disponible']")) > 0:
                continue

            product_id = product_item.xpath(".//div[@class='cart_product_img']//img/@src")[0].split('/')
            product_id = product_id[len(product_id) - 2]

            product_name = product_item.xpath(".//div[@class='cart_product_txt']/h3/a/span")[0].text
            product_name = product_name.encode('utf-8')

            product_price = float(product_item.xpath(".//p[@class='ahora']/span")[0].text.strip().\
                                  partition(' ')[0].replace(',', '.'))

            product_unitary_price = product_item.xpath(".//div[contains(concat(' ', normalize-space(@class), ' '), "
                                                       "' precio_kg ')]")
            if len(product_unitary_price) > 0:
                product_unitary_price = product_unitary_price[0].text.strip(' ()').partition(' / ')
                product_unit = product_unitary_price[2]
                product_unitary_price = float(product_unitary_price[0].partition(' ')[0].replace(',', '.'))

                # The following are fixings to normalize shop "bugs"
                if product_id == '0201030800187':
                    product_unitary_price *= 2

                product_unitary_price, product_unit = self.normalize_unitary_price(product_unitary_price, product_unit)
            else:
                product_unitary_price, product_unit = self.get_unitary_price(product_price, product_name)

            Shop.add_product(self,
                             Product(product_id, product_name, product_price, product_unitary_price, product_unit))