def read_product(url, consolidator, add_func):
product_page = load_page("https://www.italianbarber.com" + url)
if product_page is not None:
manufacturer = product_page.find("div", {"id": "tbs1"}).text.strip()
model = product_page.find("div", {"class": "product-header"}).text
model = model.replace(manufacturer + " ", "").strip()
add_func(manufacturer, model, product_page, consolidator)
def handle_product_type(url, consolidator, add_func):
time.sleep(2)
print "Loading Page: " + url
page = load_page(url)
product_blocks = pull_product_blocks_from_page(page)
for product_block in product_blocks:
read_product(product_block, consolidator, add_func)
def read_product(url, consolidator, add_func):
product_page = load_page(url)
if product_page is not None:
brand = product_page.find("h5", {"class": "brandName"}).text
model = product_page.find("h1").text
model = model.replace(brand + " ", "").strip()
add_func(brand, model, product_page, consolidator)
def handle_product_type(url, consolidator, add_func):
print "Loading Page: " + url
page = load_page(url)
urls = pull_urls_from_page(page)
for prodUrl in urls:
time.sleep(2)
read_product(prodUrl, consolidator, add_func)
next_link = check_for_next(page)
if next_link is not None:
handle_product_type(next_link, consolidator, add_func)
def handle_product_type(url, consolidator, add_func):
print "Loading Page: " + url
page = load_page(url)
links = pull_urls_from_page(page)
for link in links:
read_product(link, consolidator, add_func)
time.sleep(2)
next_link = check_for_next(page)
if next_link is not None:
handle_product_type("https://www.italianbarber.com" + next_link,
consolidator, add_func)
def handle_product_type(url, consolidator, add_func):
print "Loading Page: " + url
page = load_page(url)
urls = pull_urls_from_page(page)
for url in urls:
time.sleep(2)
read_product(url, consolidator, add_func)
next_link = check_for_next(page)
if next_link is not None:
handle_product_type("https://www.shavenation.com" + next_link,
consolidator, add_func)
def read_product(url, consolidator, add_func):
product_page = load_page("https://shavenation.com" + url)
meta_data = product_page.find("meta", {"name": "twitter:data2"})
brand = meta_data.get("content")
if brand is None or len(brand) is 0:
brand = product_page.find("h2", {"itemprop": "brand"}).text
model = product_page.find("h1", {"class": "page-title"}).text
model = model.replace(brand + " ", "").strip()
add_func(brand, model, product_page, consolidator)
def handle_product_type(url, consolidator, add_func):
print "Loading Page: " + url
page = load_page(url)
product_blocks = pull_product_blocks_from_page(page)
for product_block in product_blocks:
read_product(product_block, consolidator, add_func)
next_link = check_for_next(page)
if next_link is not None:
time.sleep(2)
handle_product_type("https://www.westcoastshaving.com" + next_link,
consolidator, add_func)
def handle_product_type(url, brand, consolidator, add_func):
time.sleep(2)
print "Loading Page: " + url
page = load_page(url)
product_blocks = pull_product_blocks_from_page(page)
for product_block in product_blocks:
read_product(product_block, brand, consolidator, add_func)
next_link = check_for_next(page)
if next_link is not None:
handle_product_type(
"http://www.phoenixartisanaccoutrements.com" + next_link, brand,
consolidator, add_func)
def read_product(url, consolidator, add_func):
product_page = load_page(url)
brand = ""
table_rows = product_page.find_all("tr")
for row in table_rows:
if "Manufacturer" in row.find("th").text:
brand = row.find("td").text
break
model = product_page.find("h1", {"itemprop": "name"}).text
model = model.replace(brand + " ", "").strip()
# print brand + " : " + model
add_func(brand, model, product_page, consolidator)