# Category Scraping
for ctg, url in urls_ctgs_dict.items():
categories[ctg] = []
fpath = fpath_namer(shop_id, 'ctg', ctg, 0)
if not op.exists(fpath):
driver.get(url)
# Get scroll height
last_height = driver.driver.execute_script(
"return document.body.scrollHeight")
while True:
# Scroll down to bottom
driver.driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
try:
driver.waitclick('//div[@class="mod_product_list__more"]/a',
timeout=5,
silent=True)
except:
pass
# Wait to load page
sleep(2)
# Calculate new scroll height and compare with last scroll height
new_height = driver.driver.execute_script(
"return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
driver.save_page(fpath)
tree = etree.parse(BytesIO(open(fpath, 'rb').read()), parser=parser)
produrl = parse_qs(urlsplit(produrl).query)['url'][0] if 'url' in parse_qs(urlsplit(produrl).query) else produrl
products[produrl] = {
'pdct_name_on_eretailer': "".join(li.xpath('.//h2[@class="product-name"]//text()')),
'raw_price': ''.join(w for t in li.xpath('.//span[@class="price"]/text()') for w in t.split()).strip(),
}
print(products[produrl], produrl)
products[produrl]['price'] = getprice(products[produrl]['raw_price'])
print(products[produrl])
categories[ctg].append(produrl)
# Going to next page if need be
next_page_click = '//a[@class="next i-next"]'
if not op.exists(fpath_namer(shop_id, 'ctg', ctg, p+1)):
if not driver.check_exists_by_xpath(next_page_click):
break
else:
driver.waitclick(next_page_click)
print(ctg, url, p, len(categories[ctg]))
# KW searches Scraping - with selenium - one page per search
search_url = "http://twinliquors.com/shop/catalogsearch/result/?q={kw}"
for kw in keywords:
searches[kw] = []
# Storing and extracting infos
fpath = fpath_namer(shop_id, 'search', kw, 0)
url = search_url.format(kw=kw, page=0)
if not op.exists(fpath):
driver.get(url)
sleep(2)
driver.save_page(fpath, scroll_to_bottom=True)
tree = etree.parse(BytesIO(open(fpath, 'rb').read()), parser=parser)
'white_wine': 'https://www.waitrose.com/ecom/shop/browse/groceries/beer_wine_and_spirits/wine/white_wine',
'gin': 'https://www.waitrose.com/ecom/shop/browse/groceries/beer_wine_and_spirits/spirits_and_liqueurs/gin',
'tequila': 'https://www.waitrose.com/ecom/shop/browse/groceries/beer_wine_and_spirits/spirits_and_liqueurs/tequila',
'rum': 'https://www.waitrose.com/ecom/shop/browse/groceries/beer_wine_and_spirits/spirits_and_liqueurs/rum',
'liquor': 'https://www.waitrose.com/ecom/shop/browse/groceries/beer_wine_and_spirits/spirits_and_liqueurs/liqueurs_and_aperitifs',
'brandy': 'https://www.waitrose.com/ecom/shop/browse/groceries/beer_wine_and_spirits/spirits_and_liqueurs/brandy',
}
# Category Scraping - with selenium - one page per category
for ctg, url in urls_ctgs_dict.items():
count = 1
categories[ctg] = []
fpath = fpath_namer(shop_id, 'ctg', ctg, 0)
if not op.exists(fpath):
driver.get(url)
driver.waitclick('//*[@class="closeNoticeSomethingDifferentPopup"]', timeout=4)
last_height = driver.driver.execute_script("return document.body.scrollHeight")
while True:
sleep(1)
driver.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
driver.waitclick('//*[@data-actiontype="load"]', timeout=3)
driver.waitclick('//*[@data-actiontype="load"]', timeout=0.5)
new_height = driver.driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
driver.save_page(fpath, scroll_to_bottom=True)
tree = etree.parse(BytesIO(open(fpath, 'rb').read()), parser=parser)
for li in tree.xpath('//article[@data-test="product-pod"]'):
produrl = clean_url(li.xpath('.//a[h2]/@href')[0], root_url)
}
# Category Scraping - with selenium - multiple pages per category (click on next page)
for ctg, url in urls_ctgs_dict.items():
categories[ctg] = []
number_of_pdcts_in_ctg = 0
if not op.exists(fpath_namer(shop_id, 'ctg', ctg, 0)):
driver.get(url)
for p in range(100):
fpath = fpath_namer(shop_id, 'ctg', ctg, p)
if not op.exists(fpath):
sleep(2)
driver.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
sleep(2)
driver.waitclick('//button[@data-dismiss="modal"]', timeout=7)
driver.save_page(fpath, scroll_to_bottom=False)
tree = etree.parse(open(fpath, 'rb'), parser=parser)
for li in tree.xpath('//article[@data-entity-type="product"]'):
produrl = li.xpath('.//a[@data-ec-linklabel="Product Text"]/@href')[0]
produrl = parse_qs(urlsplit(produrl).query)['url'][0] if 'url' in parse_qs(urlsplit(produrl).query) else produrl
products[produrl] = {
'pdct_name_on_eretailer': " ".join("".join(li.xpath('.//h1//text()')).split()),
'volume': " ".join("".join(li.xpath('.//h1//text()')).split()),
'raw_price': ''.join(w for t in li.xpath('.//*[starts-with(@class, "price") and not(contains(@class, "crossed"))]//text()') for w in t.split()).strip().split('*')[0],
'raw_promo_price': ''.join(w for t in li.xpath('.//*[starts-with(@class, "price") and contains(@class, "crossed")]//text()') for w in t.split()).strip(),
}
print(products[produrl], produrl)
tmp_price_str = products[produrl]['raw_price'].split("€")[1]
print(tmp_price_str, "€" + tmp_price_str[:-2] + ',' + tmp_price_str[-2:])
products[produrl]['price'] = getprice("€" + tmp_price_str[:-2] + ',' + tmp_price_str[-2:])
# if not r.from_cache:
# sleep(2)
print([(c, len(categories[c])) for c in categories])
# KW searches Scraping - with selenium - with search string - multiple page per search
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
search_url = "http://www.selfridges.com/GB/en/cat/foodhall/wines-spirits/"
for kw in keywords:
print("Searching", kw)
searches[kw] = []
fpath = fpath_namer(shop_id, 'search', kw, 0)
if not op.exists(fpath):
driver.get(search_url)
driver.waitclick('//*[@title="Search Selfridges..."]')
# driver.waitclick('//*[@class="searchSubmit"]')
actions = ActionChains(driver.driver)
actions.send_keys(kw)
actions.send_keys(Keys.ENTER)
actions.perform()
driver.save_page(fpath, scroll_to_bottom=True)
tree = etree.parse(open(fpath, 'rb'), parser=parser)
for li in tree.xpath('//div[@class="productsInner"]/div[not(@class="productContainerOrphan")]'):
produrl = str(li.xpath('.//a/@href')[0])
produrl = parse_qs(urlsplit(produrl).query)['url'][0] if 'url' in parse_qs(
urlsplit(produrl).query) else produrl
produrl = clean_url(produrl, root_url)
products[produrl] = {
'pdct_name_on_eretailer': ' '.join(' '.join(li.xpath('.//div[@class="productContainerDesc"]/a//text()')).split()).strip(),
'white_wine': 'https://www.hawesko.de/weisswein',
}
# Category Scraping - with selenium - one page per category
for ctg, url in urls_ctgs_dict.items():
categories[ctg] = []
fpath = fpath_namer(shop_id, 'ctg', ctg, 0)
if not op.exists(fpath):
driver.get(url)
for k in range(20):
sleep(1.5)
if driver.check_exists_by_xpath(
'//div[@class="article list loader"]//*[@class="button loading loaderbutton"]'
):
driver.waitclick(
'//div[@class="article list loader"]//*[@class="button loading loaderbutton"]'
)
sleep(1)
else:
break
driver.save_page(fpath)
tree = etree.parse(BytesIO(open(fpath, 'rb').read()), parser=parser)
for li in tree.xpath('//div[@data-module="article"]'):
if not li.xpath('.//div/a/@href'):
break
produrl = li.xpath('.//div/a/@href')[0]
produrl = parse_qs(
urlsplit(produrl).query)['url'][0] if 'url' in parse_qs(
urlsplit(produrl).query) else produrl
produrl = clean_url(produrl, root_url)
'raw_promo_price': ''.join(w for t in li.xpath('.//*[@class="thumbnail-priceOld"]//text()')[:3] for w in t.split()).strip(),
}
print(products[produrl], produrl)
if products[produrl]['raw_price'].count('$') >= 2:
products[produrl]['raw_price'] = ''.join(w for t in li.xpath('.//*[@class="thumbnail-price promo"]//text()')[:3] for w in t.split()).strip()
products[produrl]['price'] = getprice(products[produrl]['raw_price'])
products[produrl]['promo_price'] = getprice(products[produrl]['raw_promo_price'])
print(products[produrl])
categories[ctg].append(produrl)
# Checking if it was the last page
if len(set(categories[ctg])) == number_of_pdcts_in_ctg:
break
else:
number_of_pdcts_in_ctg = len(set(categories[ctg]))
driver.waitclick('//*[class="glyphicon glyphicon-menu-right"]')
print([(c, len(categories[c])) for c in categories])
# KW searches Scraping - with requests - one page per search
kw_search_url = "https://www.macave.leclerc/catalogsearch/result/?q={kw}"
for kw in keywords:
print('Requesting', kw)
searches[kw] = []
fpath = fpath_namer(shop_id, 'search', kw, 0)
if not op.exists(fpath):
driver.get(kw_search_url.format(kw=kw))
driver.waitclick('//div[@class="limiter"]/span[last()]')
sleep(2)
driver.save_page(fpath, scroll_to_bottom=True)
'raw_price':
" ".join("".join(
li.xpath(
'.//span[@class="price"]/span[last()]//text()|.//span[@class="wasprice"]/span[2]//text()'
)[:1]).split()),
}
print(products[produrl])
products[produrl]['price'] = getprice(
products[produrl]['raw_price'])
print(products[produrl])
if len(set(categories[ctg])) == number_of_pdcts_in_ctg:
break
else:
number_of_pdcts_in_ctg = len(set(categories[ctg]))
if scraping_launched:
driver.waitclick(
'//a[contains(@class, "forward-listing btn")]')
sleep(1)
print([(c, len(categories[c])) for c in categories])
# Easy case, where you scroll down to get the whole page
search_url = "https://groceries.asda.com/search/{kw}"
for kw in keywords:
print(kw)
searches[kw] = []
# Storing and extracting infos
fpath = fpath_namer(shop_id, 'search', kw, 0)
url = search_url.format(kw=kw, page=0)
if not op.exists(fpath):
driver.get(url)
sleep(3)
driver.save_page(fpath, scroll_to_bottom=True)
produrl = tr.xpath('.//*[contains(@class, "prodstitle")]/@href')[0]
produrl = parse_qs(urlsplit(produrl).query)['url'][0] if 'url' in parse_qs(urlsplit(produrl).query) else produrl
produrl = clean_url(produrl, root_url)
products[produrl] = {
'volume': tr.xpath('.//*[contains(@class, "prodstitle")]/@title')[0],
'pdct_name_on_eretailer': tr.xpath('.//*[contains(@class, "prodstitle")]/@title')[0],
'raw_price': ''.join(tr.xpath('.//span[contains(@class, "prodsprice")]/text()')).strip(),
'raw_promo_price': ''.join(tr.xpath('.//span[contains(@class, "prodsprice")]/s/text()')).strip(),
}
print(products[produrl], produrl)
products[produrl]['price'] = getprice(products[produrl]['raw_price'])
products[produrl]['promo_price'] = getprice(products[produrl]['raw_promo_price'])
print(products[produrl])
categories[ctg].append(produrl)
if req_sent and not driver.waitclick('//div[contains(@class, "c data2")]/table[last()]//a[contains(text(),"{page}")]'.format(page=page+1)):
break
print(ctg, len(categories[ctg]))
######################################
# # KW searches scrapping ############
######################################
# KW searches Scraping - with requests - one page per search
kw_search_url = 'https://www.b-21.com/searchprods.asp?searchstring={kw}&pagenumber={page}&val=0'
for kw in keywords:
searches[kw] = []
number_of_pdcts_in_kw_search = 0
req_sent = False
# Category Scraping - with selenium - multiple pages per category (click on next page)
for ctg, url in urls_ctgs_dict.items():
categories[ctg] = []
number_of_pdcts_in_ctg = 0
fpath = fpath_namer(shop_id, 'ctg', ctg, 0)
if not op.exists(fpath):
driver.get(url)
click_trials = 0
while True:
driver.scroll_to_bottom()
sleep(2)
if driver.wait_for_xpath(
'//a[@class="btn load-products loading-button externalLink"]'
):
driver.waitclick(
'//a[@class="btn load-products loading-button externalLink"]'
)
click_trials += 1
if click_trials > 1:
break
else:
break
driver.save_page(fpath)
tree = etree.parse(open(fpath, 'rb'), parser=parser)
# for li in tree.xpath('//li[contains(@id,"WC_CatalogSearchResultDisplay")]'):
for li in tree.xpath('//div[@class="item isUpdated"]'):
if not li.xpath('.//figure/a/@href'):
continue
# produrl = li.xpath('./a/@href')[0]
produrl = li.xpath('.//figure/a/@href')[0]
'raw_price': ' '.join(''.join(li.xpath('.//span[@class="RegularPrice"]/text()')).split()),
'raw_promo_price': ' '.join(''.join(li.xpath('.//span[@class="cart-price-strike"]//text()')).split()),
}
print(products[produrl], produrl)
products[produrl]['price'] = getprice(products[produrl]['raw_price'])
products[produrl]['promo_price'] = getprice(products[produrl]['raw_promo_price'])
print(products[produrl])
categories[ctg].append(produrl)
# Checking if it was the last page
if len(set(categories[ctg])) == number_of_pdcts_in_ctg:
break
else:
number_of_pdcts_in_ctg = len(set(categories[ctg]))
driver.waitclick('//b[contains(text(),">>")]')
# KW searches Scraping - with selenium - with nb page hard-coded in url - multiple page per search
search_url = "https://buckhead.towerwinespirits.com/main.asp?request=SEARCH&search={kw}"
for kw in keywords:
searches[kw] = []
number_of_pdcts_in_kw_search = 0
# Storing and extracting infos
urlp = search_url.format(kw=kw)
fpath = fpath_namer(shop_id, 'search', kw, 0)
if not op.exists(fpath):
driver.get(urlp)
sleep(1)
driver.save_page(fpath, scroll_to_bottom=True)
if not op.exists(fpath):
if not minibardelivery_was_initialised:
init_minibardelivery(driver)
minibardelivery_was_initialised = True
driver.get(url)
# Get scroll height
last_height = driver.driver.execute_script(
"return document.body.scrollHeight")
while True:
# Scroll down to bottom
driver.driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
try:
driver.waitclick(
'//*[contains(@class, "product-list-load-more")]',
timeout=6)
except:
pass
# Wait to load page
sleep(2)
# Calculate new scroll height and compare with last scroll height
new_height = driver.driver.execute_script(
"return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
driver.save_page(fpath)
# Parsing
"whisky": "https://www.lagrandeepicerie.com/fr/cave/spiritueux/whiskies/",
"cognac": "https://www.lagrandeepicerie.com/fr/cave/spiritueux/cognac/",
"vodka": "https://www.lagrandeepicerie.com/fr/cave/spiritueux/vodka/",
"rum": "https://www.lagrandeepicerie.com/fr/cave/spiritueux/rhums/",
"gin": "https://www.lagrandeepicerie.com/fr/cave/spiritueux/gin/",
}
# Category Scraping
for ctg, url in urls_ctgs_dict.items():
categories[ctg] = []
fpath = fpath_namer(shop_id, 'ctg', ctg, 0)
if not op.exists(fpath):
driver.get(url)
sleep(2)
driver.waitclick('//*[@id="show-more-product"]', timeout=5)
# driver.click_to_bottom('//*[@id="show-more-product"]')
driver.save_page(fpath, scroll_to_bottom=True)
tree = etree.parse(BytesIO(open(fpath, 'rb').read()), parser=parser)
print(ctg, fpath)
for li in tree.xpath('//*[@id="search-result-items"]/div'):
produrl = li.xpath('.//a[@class="thumb-link"]/@href')[0]
produrl = parse_qs(
urlsplit(produrl).query)['url'][0] if 'url' in parse_qs(
urlsplit(produrl).query) else produrl
produrl = clean_url(produrl, root_url)
products[produrl] = {
'pdct_name_on_eretailer':
' '.join(''.join(li.xpath(
'.//div[@class="product-name"]//text()')).split()).strip(),
'raw_price':