def parse_page(self, driver, url): product = {'url': url, 'brand': ProductDgSpider.brand, 'code': '', 'price': 0, 'images': '', 'detail': ''} # check 404 element = utils.find_element_by_css_selector(driver, 'div.b-error_page-wrapper') if element: raise Exception('404 page not found') # Switch to CHN element = utils.find_elements_by_css_selector(driver, 'ul.l-header_service_menu > li > span')[0] driver.execute_script('arguments[0].click();', element) utils.sleep(1) element = utils.find_elements_by_css_selector(driver, 'ul.b-language_selector-language_list > li > a')[-1] driver.execute_script('arguments[0].click();', element) utils.sleep(1) # title element = utils.find_element_by_css_selector(driver, 'span.b-product_name') if element: product['title'] = element.text.strip() else: raise Exception('Title not found for %s' % driver.current_url) # code element = utils.find_element_by_css_selector(driver, 'div.b-product_master_id') product['code'] = element.get_attribute('innerHTML').split(':')[1].strip() # unit product['unit'] = config.rmb # price element = utils.find_element_by_css_selector(driver, 'h4.b-product_price-standard') if element: price_text = element.text.strip()[1:].replace(',', '') try: product['price'] = float(price_text) except: pass # 有那么几件商品没价格 # images images = [] elements = utils.find_elements_by_css_selector(driver, 'div.js-thumbnails_slider > ul.js-thumbnails > li > img') for element in elements: text = element.get_attribute('src').strip() images.append(text.split('?')[0]) product['images'] = ';'.join(images) # detail element = utils.find_element_by_css_selector(driver, 'div.b-product_long_description') text = element.get_attribute('innerHTML') text = text[1:] # Remove the first '\n' text = text.replace('amp;', '') text = text.replace('。', '。\n') text = text.replace('<i>', '') text = text.replace('</i>', '') text = text.replace('<br>', '') text = text.replace('• ', '\n• ') product['detail'] = text[:-2] # Remove the tailing '\n' return product
def parse_page(self, driver, url): products = [] elements = utils.find_elements_by_css_selector( driver, 'div.productgridItem > ul.prod_style > li[style="display:block"] > div.prod_grid > a' ) for element in elements: products.append(element.get_attribute('href').strip()) return ';'.join(products)
def parse_page(self, driver, url): products = [] driver.get(url) product = { 'url': url, 'brand': ProductStuartweitzmanSpider.brand, 'code': '', 'price': 0, 'images': '', 'detail': '' } # check 404 element = utils.find_element_by_css_selector( driver, 'div.information_message.negative > p') if element: if element.text.strip() == '404 页面未找到': raise Exception('404 page not found') # title element = utils.find_element_by_css_selector(driver, 'h1.pdname > span') if element: product['title'] = element.text.strip() else: raise Exception('Title not found for %s' % driver.current_url) # code/NA # unit product['unit'] = config.rmb # price element = utils.find_element_by_css_selector(driver, 'p.big-price') if element: price_text = element.text.strip() # 打折的情况需要处理 if '|' in price_text: # 打折 price_text = price_text.split('|')[1].strip().split(' ')[0][1:] else: price_text = price_text[1:-2].strip() price_text = price_text.replace(',', '') product['price'] = float(price_text) # images elements = utils.find_elements_by_css_selector( driver, 'ul#carousel_alternate > li > span > a > img') images = [ element.get_attribute('data-primaryimagesrc').strip() for element in elements ] product['images'] = ';'.join(images) # detail element = utils.find_element_by_css_selector(driver, 'div.pdp-description') if element: product['detail'] = element.text.strip() return product
def parse_page(self, driver, url): products = [] product_count = 0 while True: elements = utils.find_elements_by_css_selector(driver, 'div.l-product_tiles > div.js-product_tile > div > a.js-producttile_link') if len(elements) > product_count: product_count = len(elements) driver.execute_script('window.scrollBy(0, document.body.scrollHeight);') utils.sleep(1) else: break for element in elements: products.append(element.get_attribute('href').strip()) return ';'.join(products)
def parse_page(self, driver, url): products = [] product = { 'url': url, 'brand': ProductFerragamoSpider.brand, 'code': '', 'price': 0, 'images': '', 'detail': '' } # check 404 element = utils.find_element_by_css_selector(driver, 'div.nofound') if element: raise Exception('404 page not found') # title element = utils.find_element_by_css_selector( driver, 'div.dpd-main__details__head > div > h1.dpd-main__name') if element: product['title'] = element.text.strip() else: raise Exception('Title not found for %s' % driver.current_url) # code element = utils.find_element_by_css_selector( driver, 'div.dpd-main__details__head > div > div.dpd-main__sku') if element: product['code'] = ' '.join(element.text.strip().split(' ')[1:]) # unit product['unit'] = config.rmb # price element = utils.find_element_by_css_selector( driver, 'div.dpd-main__details__head > div > div.dpd-main__price') if element: product['price'] = float(element.text.strip().split(' ')[1]) # images elements = utils.find_elements_by_css_selector( driver, 'div.dpd-visuals > div > a > img') images = [element.get_attribute('src').strip() for element in elements] product['images'] = ';'.join(images) # detail element = utils.find_element_by_css_selector(driver, 'div.dpd-info__body') if element: product['detail'] = element.get_attribute('innerHTML').strip() return product
def parse_page(self, driver, url): # 点击展开分页 element = utils.find_element_by_css_selector( driver, 'li.view-all-products > span') if element: driver.execute_script('arguments[0].click();', element) # 下拉刷新 products = [] product_count = 0 while True: elements = utils.find_elements_by_css_selector( driver, 'div.product-tile > figure > a.thumb-link') if len(elements) > product_count: product_count = len(elements) driver.execute_script( 'window.scrollBy(0, document.body.scrollHeight);') utils.sleep(1) else: break for element in elements: products.append(element.get_attribute('href').strip()) return ';'.join(products)
def parse_page(self, driver, url): products = [] product = {'url': url, 'brand': ProductLoeweSpider.brand, 'code': '', 'price': 0, 'images': '', 'detail': ''} # 切换语言 element = utils.find_element_by_css_selector(driver, 'div.siteSelectors-current.siteSelectors-current-locale') if element: driver.execute_script('arguments[0].click();', element) utils.sleep(1) elements = utils.find_elements_by_css_selector(driver, 'ul.siteSelectors-list-locale[data-country=CN] > li > a') switch_to_chinese = False for element in elements: if element.get_attribute('innerHTML').strip() == 'Simplified Chinese': driver.execute_script('arguments[0].click();', element) utils.sleep(1) switch_to_chinese = True break if not switch_to_chinese: raise Exception("Can't switch to Chinese") else: raise Exception("Cant't select language") # check 404 element = utils.find_element_by_css_selector(driver, 'div.error404') if element: raise Exception('404 page not found') # title element = utils.find_element_by_css_selector(driver, 'h1.product-name') if element: product['title'] = element.text.strip() else: raise Exception('Title not found for %s' % driver.current_url) # code element = utils.find_element_by_css_selector(driver, 'span.model-id') if element: product['code'] = element.text.strip().split(': ')[1] # unit product['unit'] = config.rmb # price element = utils.find_element_by_css_selector(driver, 'div.price-and-size-wrapper > div.product-price > span.price-sales') if element: if element.text.strip() != '不适用': price_text = element.text.strip()[1:].replace(',', '') product['price'] = float(price_text) # images elements = utils.find_elements_by_css_selector(driver, 'ul.product-thumbnails-list > li > a') images = [element.get_attribute('href').strip() for element in elements] product['images'] = ';'.join(images) # detail details = [] pattern = re.compile(r'<[^>]+>',re.S) elements = utils.find_elements_by_css_selector(driver, 'ul.details-col-1 > li') for element in elements: text = element.get_attribute('innerHTML').strip() text = pattern.sub('', text) # 去除HTML标签 text = text.replace('\n\t\t', '') details.append(text) elements = utils.find_elements_by_css_selector(driver, 'ul.details-col-2 > li') for element in elements: text = element.get_attribute('innerHTML').strip() text = pattern.sub('', text) text = text.replace('\n\t\t', '') details.append(text) product['detail'] = '\n'.join(details) return product