def _get_parsed_product_from_url(self, url) -> Union[None, ParsedProduct]: page_source = self._load_page_with_TL(url) if page_source is None: # fixme - log - fatal - can't load page print( f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}" ) return None soup = BeautifulSoup(page_source, 'html.parser') parsed_product = get_empty_parsed_product_dict() parsed_product['url'] = url # title title = remove_odd_space( soup.find('h1', class_='product-cart__title').text) sub_title = remove_odd_space( soup.find('a', class_='product-cart__content-info-header-black').text) title += ' ' + sub_title parsed_product['title'] = title # price price = remove_ALL_spaces( soup.find('div', class_='product-cart__content-price-actual').text)[:-1] parsed_product['price_new'] = price return parsed_product
def _get_parsed_product_from_url(self, url) -> Union[None, ParsedProduct]: page_source = self._load_page_with_TL(url, 10.0) if page_source is None: # fixme - log - fatal - can't load page print( f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}" ) return None soup = BeautifulSoup(page_source, 'html.parser') parsed_product = get_empty_parsed_product_dict() parsed_product['url'] = url # title title = remove_odd_space(soup.find('h1', class_='b-offer-title').text) parsed_product['title'] = title # price price = remove_non_digits( soup.find('div', class_='b-offer-box__price').text) parsed_product['price_new'] = price return parsed_product
def _get_parsed_product_from_url(self, url) -> Union[None, ParsedProduct]: page_source = self._load_page_with_TL(url) if page_source is None: # fixme - log - fatal - can't load page print(f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}") return None soup = BeautifulSoup(page_source, 'html.parser') parsed_product = get_empty_parsed_product_dict() parsed_product['url'] = url # title # title title = remove_odd_space(soup.find('div', class_='range-revamp-header-section__title--big').text) sub_title = remove_odd_space(soup.find('span', 'range-revamp-header-section__description-text').text) title += ' ' + sub_title parsed_product['title'] = title # price price = remove_ALL_spaces(soup.find('span', class_='range-revamp-price__integer').text) parsed_product['price_new'] = price return parsed_product
def _get_parsed_product_from_search( self, category_row) -> Union[None, List[ParsedProduct]]: if category_row['sub_type'] != 'appliances': return None parsed_product_list = [] url = self.make_search_url(category_row['search_word']) print(f"{self.get_handler_name()} -> {category_row['cat_title']}") print(f'using url:\n{url}') page_source = self._load_page_with_TL(url, 10.0) if page_source is None: # fixme - log - fatal - can't load page print( f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}" ) return None soup = BeautifulSoup(page_source, 'html.parser') for parsed_item in soup.find_all('div', class_='b-product-block'): try: parsed_product = get_empty_parsed_product_dict() # title title = remove_odd_space( parsed_item.find('div', class_='b-product-block__name').text) try: sub_title = remove_odd_space( parsed_item.find('div', class_='b-product-block__type').text) title = sub_title + ' ' + title except: pass parsed_product['title'] = title # url url = parsed_item.find( 'a', class_='b-product-block__main-link')['href'] url = fr'https://www.svyaznoy.ru{url}' parsed_product['url'] = url # price price = remove_non_digits( parsed_item.find( 'span', class_='b-product-block__visible-price').text) parsed_product['price_new'] = price parsed_product_list.append(parsed_product) except: # FIXME log fatal print("can't parse svaznoy item") print(parsed_item) return parsed_product_list
def _get_parsed_product_from_search( self, category_row) -> Union[None, List[ParsedProduct]]: if category_row['sub_type'] != 'appliances': return None full_parsed_product_list = [] for page_num in range(1): parsed_product_list = [] # url = self.get_search_url_for_category(category_row, page_num) url = self.create_general_search_url(category_row['search_word'], page_num) print(f"{self.get_handler_name()} -> {category_row['cat_title']}") print(f'using url:\n{url}') page_source = self._load_page_with_TL(url) if page_source is None: # fixme - log - fatal - can't load page print( f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}" ) return None soup = BeautifulSoup(page_source, 'html.parser') for parsed_item in soup.find_all('li', {'data-dy': 'product'}): parsed_product = get_empty_parsed_product_dict() # title title = remove_odd_space( parsed_item.find('a', { 'data-dy': 'title' }).text) parsed_product['title'] = title # url url = remove_odd_space( parsed_item.find('a', {'data-dy': 'title'})['href']) url = f"https://www.eldorado.ru{url}" parsed_product['url'] = url # price price_list = [] for price_item in parsed_item.find_all('span'): if hasattr(price_item, 'text') and price_item.text[-2:] == 'р.': mb_price = float(remove_non_digits(price_item.text)) if 100 <= mb_price <= 100000: price_list.append(mb_price) price = sorted(price_list)[-1] parsed_product['price_new'] = price parsed_product_list.append(parsed_product) full_parsed_product_list.extend(parsed_product_list) return full_parsed_product_list
def _get_parsed_product_from_search(self, category_row) -> Union[None, List[ParsedProduct]]: if category_row['type'] != 'food': return None parsed_product_list = [] url = self._create_serch_url_for_category(category_row) print(f"{self.get_handler_name()} -> {category_row['cat_title']}") page_source = self._load_page_with_TL(url, 10.0) if page_source is None: # fixme - log - fatal - can't load page print(f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}") return [] if 'товар не представлен' in str(page_source): # FIXME error log print(f"no searched item in {self.get_handler_name()}, {category_row['cat_title']}") raise ValueError('no searched item in shop') soup = BeautifulSoup(page_source, 'html.parser') for page_item in soup.find_all('div', class_='sku-card-small-container'): parsed_product = get_empty_parsed_product_dict() try: # title title = page_item.find('div', class_='sku-card-small__title').text try: # sub title sub_title = page_item.find('div', class_='sku-card-small__sub-title').text title += ' ' + sub_title except: pass parsed_product['title'] = title # url url = page_item.find('a', class_='sku-card-small')['href'] parsed_product['url'] = fr"https://lenta.com{url}" # price price_new = page_item.find('span', class_='sku-price__integer').text parsed_product['price_new'] = price_new parsed_product['price_old'] = None parsed_product_list.append(parsed_product) except: print('ERROR! in parsing page_item') with open(f"_{self.get_handler_name()}_{category_row['cat_title']}_{time.time()}.page_item", 'w+') as file: file.write(str(page_item)) return parsed_product_list
def _get_parsed_product_from_search( self, category_row) -> Union[None, List[ParsedProduct]]: if category_row['type'] != 'food': return None parsed_product_list = [] url = self._create_serch_url_for_category(category_row['search_word']) print(f"{self.get_handler_name()} -> {category_row['cat_title']}") print(f'using url:\n{url}') page_source = self._load_page_with_TL(url, 10.0) if page_source is None: # fixme - log - fatal - can't load page print( f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}" ) return None soup = BeautifulSoup(page_source, 'html.parser') for parsed_item in soup.find_all('li', class_='xf-catalog__item'): if "Временно отсутствует" in str(parsed_item): continue parsed_product = get_empty_parsed_product_dict() # title title = remove_odd_space( parsed_item.find('a', class_='xf-product-title__link').text) parsed_product['title'] = title # url url = parsed_item.find('a', class_='xf-product-title__link')['href'] url = f"https://perekrestok.ru{url}" parsed_product['url'] = url # price try: price = parsed_item.find( 'div', class_='xf-product-cost__old-price')['data-cost'] except: price = parsed_item.find( 'div', class_='xf-product-cost__current')['data-cost'] parsed_product['price_new'] = price parsed_product_list.append(parsed_product) return parsed_product_list
def _get_parsed_product_from_search( self, category_row) -> Union[None, List[ParsedProduct]]: if category_row['sub_type'] != 'medicine': return None parsed_product_list = [] url = self._create_search_url_for_category(category_row['search_word']) print(f"{self.get_handler_name()} -> {category_row['cat_title']}") print(f'using url:\n{url}') page_source = self._load_page_with_TL(url, 10.0) if page_source is None: # fixme - log - fatal - can't load page print( f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}" ) return None soup = BeautifulSoup(page_source, 'html.parser') for parsed_item in soup.find_all('div', class_='product'): try: parsed_product = get_empty_parsed_product_dict() # title title = remove_odd_space( parsed_item.find('a', class_='product__title').text) sub_title = remove_odd_space( parsed_item.find('a', class_='product-brand__link').text) title += ' ' + sub_title parsed_product['title'] = title # url url = parsed_item.find('a', class_='product__title')['href'] parsed_product['url'] = self._create_link_to_product(url) # price price = remove_ALL_spaces( parsed_item.find( 'span', class_='product__active-price-number').text) parsed_product['price_new'] = price parsed_product_list.append(parsed_product) except: # FIXME log fatal print("can't parse rigla item") print(parsed_item) return parsed_product_list
def _get_parsed_product_from_url(self, url) -> Union[None, ParsedProduct]: page_source = self._load_page_with_TL(url) if page_source is None: # fixme - log - fatal - can't load page print( f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}" ) return None soup = BeautifulSoup(page_source, 'html.parser') parsed_product = get_empty_parsed_product_dict() parsed_product['url'] = url # title parsed_product['title'] = soup.find('h1', class_='main_header').text # price price_item = soup.find('span', class_='product-price') price_new = find_float_number( price_item.find('span', class_='price').text) parsed_product['price_new'] = price_new try: price_old = find_float_number( price_item.find('span', class_='crossed').text) parsed_product['price_old'] = price_old except: # we just have now any discount parsed_product['price_old'] = None pass # units try: unit_item = soup.find('ul', class_='widget-list').find_all( 'li', class_='attributes__item')[1] unit_title = wspex( unit_item.find('div', class_='attributes__name').text) parsed_product['unit_title'] = unit_title unit_value = find_float_number( unit_item.find('div', class_='attributes__value').text) parsed_product['unit_value'] = unit_value parsed_product['unparsed_units'] = unit_value + " " + unit_title except: pass return parsed_product
def _get_parsed_product_from_search(self, category_row) -> Union[None, List[ParsedProduct]]: if category_row['sub_type'] != 'furniture': return None parsed_product_list = [] url = self._create_search_url_for_category(category_row['search_word']) print(f"{self.get_handler_name()} -> {category_row['cat_title']}") print(f'using url:\n{url}') page_source = self._load_page_with_TL(url) if page_source is None: # fixme - log - fatal - can't load page print(f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}") return None soup = BeautifulSoup(page_source, 'html.parser') for parsed_item in soup.find_all('div', class_='serp-grid__item'): try: parsed_product = get_empty_parsed_product_dict() # title title = remove_odd_space(parsed_item.find('div', 'range-revamp-header-section__title--small').text) sub_title = remove_odd_space(parsed_item.find('span', 'range-revamp-header-section__description-text').text) title += ' ' + sub_title parsed_product['title'] = title # url url = parsed_item.find('a')['href'] parsed_product['url'] = url # price price = remove_ALL_spaces(parsed_item.find('span', class_='range-revamp-price__integer').text) parsed_product['price_new'] = price parsed_product_list.append(parsed_product) except: # FIXME log fatal print("can't parse IKEA item") print(parsed_item) return parsed_product_list
def _get_parsed_product_from_url(self, url) -> Union[None, ParsedProduct]: page_source = self._load_page_with_TL(url, 10.0) if page_source is None: # fixme - log - fatal - can't load page print(f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}") return None if 'В выбранном Вами магазине данный товар не представлен' in str(page_source): # FIXME error log print('no searched item in shop') raise ValueError('no searched item in shop') soup = BeautifulSoup(page_source, 'html.parser') parsed_product = get_empty_parsed_product_dict() parsed_product['url'] = url # title title = remove_odd_space(str(soup.find('h1', class_='sku-page__title').text)) try: sub_title = remove_odd_space(soup.find('div', class_='sku-page__sub-title').text) title += ' ' + sub_title except: pass parsed_product['title'] = title # price for item in soup.find_all('div', class_='sku-prices-block__item'): if 'обычная' in str(item).lower(): price = remove_odd_space(item.find('span', class_='sku-price__integer').text).replace(' ', '') parsed_product['price_new'] = float(price.replace(',', '.')) # unit for item in soup.find_all('div', class_='sku-card-tab-params__item'): if 'Упаковка' in str(item): unit = remove_odd_space(item.find('dd', 'sku-card-tab-params__value').text) parsed_product['unparsed_units'] = unit return parsed_product
def _get_parsed_product_from_url(self, url) -> Union[None, ParsedProduct]: page_source = self._load_page_with_TL(url, 10.0) if page_source is None: # fixme - log - fatal - can't load page print( f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}" ) return None soup = BeautifulSoup(page_source, 'html.parser') parsed_product = get_empty_parsed_product_dict() parsed_product['url'] = url # title try: title = remove_odd_space( soup.find('h1', class_='xf-product-card__title').text) except: title = remove_odd_space( soup.find('h1', class_='xf-product-new__title').text) title = remove_odd_space(title) parsed_product['title'] = title # price price_new = soup.find('span', class_='js-price-rouble').text price_new = remove_odd_space(price_new) parsed_product['price_new'] = price_new try: price_old = soup.find('span', class_='js-old-price-rouble').text price_old = remove_odd_space(price_old) except: price_old = None parsed_product['price_old'] = price_old return parsed_product
def _get_parsed_product_from_url(self, url) -> Union[None, ParsedProduct]: page_source = self._load_page_with_TL(url) if page_source is None: # fixme - log - fatal - can't load page print( f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}" ) return None soup = BeautifulSoup(page_source, 'html.parser') parsed_product = get_empty_parsed_product_dict() parsed_product['url'] = url # title title = remove_odd_space( soup.find('h1', class_='catalogItemDetailHd', itemprop='name').text) parsed_product['title'] = title # price try: price = remove_ALL_spaces( soup.find('span', class_='product-box-price__old-el').text)[:-2] except: price = remove_ALL_spaces( soup.find('div', class_='product-box-price__active').text)[:-2] parsed_product['price_new'] = price parsed_product['price_old'] = None # float, value in unit of unit_title parsed_product['unit_value'] = 1 # string, name of units parsed_product['unit_title'] = '1шт' return parsed_product
def _get_parsed_product_from_search( self, categoty_row) -> Union[None, List[ParsedProduct]]: if categoty_row['type'] != 'food': return None parsed_product_list = [] url = self._create_serch_url_for_category( str(categoty_row['search_word']).replace(' ', '+')) print(f"{self.get_handler_name()} -> {categoty_row['cat_title']}") print(f'using url:\n{url}') page_source = self._load_page_with_TL(url) if page_source is None: # fixme - log - fatal - can't load page print( f"can't load page, info:\n, handler : {self.get_handler_name()}\nurl: {url}" ) return None soup = BeautifulSoup(page_source, 'html.parser') for product_list in soup.find_all('div', class_='product_listing_container'): for product_item in product_list.find_all('div', class_='product'): try: parsed_product = get_empty_parsed_product_dict() # title parsed_product['title'] = product_item.find('a')['title'] # url parsed_product[ 'url'] = rf"https://www.okeydostavka.ru{product_item.find('a')['href']}" try: parsed_product['unit_value'] = find_float_number( product_item.find('div', class_='product-weight').text) parsed_product['unit_title'] = wspex( product_item.find( 'div', class_='product-weight').find('span').text) except: parsed_product['unit_value'] = None parsed_product['unit_title'] = None # price, also work for reduced price product_item_price = product_item.find( 'div', class_='product-price') parsed_product['price_new'] = find_float_number( product_item_price.find('span', class_='price').text) # price, old (not reduced) try: old_price = wspex( product_item_price.find('span', class_='crossed').text) parsed_product['price_old'] = find_float_number( old_price) except: # just no discount parsed_product['price_old'] = None pass parsed_product_list.append(parsed_product) except: print('\nOkey parser, cant parse:') print(product_item, end='\n\n') return parsed_product_list