def met_scrape_homepage():
# headless Chrome
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1920x1080")
chrome_driver = os.getcwd() + "\\chromedriver.exe"
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path=chrome_driver)
driver.implicitly_wait(5)
# go to website
driver.get('https://www.metropcs.com')
time.sleep(5)
html = driver.page_source
soup = BeautifulSoup(html, "html.parser")
# make object
scraped_promotion = ScrapedPromotion()
# set hardcoded variables
scraped_promotion.provider = 'metropcs'
scraped_promotion.date = datetime.date.today()
scraped_promotion.time = datetime.datetime.now().time()
scraped_promotion.device_storage = '0'
scraped_promotion.device_name = 'N/A'
scraped_promotion.url = driver.current_url
scraped_promotion.promo_location = 'homepage'
# get first banner
for div in soup.findAll('div', class_='row'):
deals_page_promo = div.text.strip().replace('\n', '')
scraped_promotion.promo_text = deals_page_promo
print(scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage,
scraped_promotion.promo_location, scraped_promotion.promo_text,
scraped_promotion.url, scraped_promotion.date,
scraped_promotion.time)
add_scraped_promotions_to_database(
scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
driver.close()
def tmo_scrape_postpaid_promotions(driver, soup, url, device_name, device_storage):
# make object
scraped_promotion = ScrapedPromotion()
# set variables already determined
scraped_promotion.provider = 'tmobile'
scraped_promotion.device_name = device_name
scraped_promotion.device_storage = device_storage
scraped_promotion.url = url
# make empty list of promotions
promotions = []
# upper banner text
try:
upper_banner_text = driver.find_element_by_id('promo-banner')
promotions.append(['upper banner', upper_banner_text.text])
except NoSuchElementException:
upper_banner_text = ''
# banner under device name text
for div2 in soup.findAll("div", class_="text-magenta ng-scope"):
promotions.append(['banner under device name', div2.text])
# crossed out text (if savings is anything other than $0.00)
strike_out_price = soup.findAll('span', class_='text-magenta ng-binding')
if strike_out_price[0].text != '($0.00 Savings)':
promotions.append(['discount', strike_out_price[0].text])
# make object for each promo text instance
for promo_instance in promotions:
scraped_promotion.promo_location = promo_instance[0]
scraped_promotion.promo_text = promo_instance[1]
# time variables
scraped_promotion.date = datetime.date.today()
scraped_promotion.time = datetime.datetime.now().time()
# add to database
add_scraped_promotions_to_database(scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
def cri_scrape_homepage():
# headless Chrome
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1920x1080")
chrome_driver = os.getcwd() + "\\chromedriver.exe"
driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=chrome_driver)
driver.implicitly_wait(5)
# go to website
driver.get('https://www.cricketwireless.com/')
time.sleep(3)
html = driver.page_source
soup = BeautifulSoup(html, "html.parser")
# make object
scraped_promotion = ScrapedPromotion()
# set hardcoded variables
scraped_promotion.provider = 'cricket'
scraped_promotion.date = datetime.date.today()
scraped_promotion.time = datetime.datetime.now().time()
scraped_promotion.promo_location = 'homepage'
scraped_promotion.device_name = 'N/A'
scraped_promotion.device_storage = '0'
scraped_promotion.url = driver.current_url
# get slideshow
main = soup.find('div', class_='main')
for div1 in main.findAll('div', class_='constrain-width-1024'):
deals_page_promo = format_promo_text(div1.text)
scraped_promotion.promo_text = deals_page_promo
print(scraped_promotion.provider, scraped_promotion.device_name, scraped_promotion.device_storage,
scraped_promotion.promo_location, scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
add_scraped_promotions_to_database(scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url, scraped_promotion.date,
scraped_promotion.time)
driver.close()
def spr_scrape_postpaid_promotions(soup, url, device_name, device_storage):
# make object
scraped_promotion = ScrapedPromotion()
# set variables already determined
scraped_promotion.provider = 'sprint'
scraped_promotion.device_name = device_name
scraped_promotion.device_storage = device_storage
scraped_promotion.url = url
# make empty list of promotions
promotions = []
# promotion text under price box
price_boxes = soup.findAll('div', class_='col-xs-24 col-lg-24 col-xl-24 mb-20 active')
for box in price_boxes:
if box.find('strong', class_='display-block font-size-16 font-size-md-18 color--blue').text != ' Full price':
for li in box.findAll('li', class_='promo-item'):
promotions.append(['banner under Sprint Flex box', li.text.replace('Detail', '').strip()])
# crossed out price
strike_out_price = soup.findAll('del', class_='sprint-price-del')
if strike_out_price[0].text != '':
promotions.append(['crossed out price', strike_out_price[0].text])
# make object for each promo text instance
for promo_instance in promotions:
scraped_promotion.promo_location = promo_instance[0]
scraped_promotion.promo_text = promo_instance[1]
# time variables
scraped_promotion.date = datetime.date.today()
scraped_promotion.time = datetime.datetime.now().time()
# add to database
add_scraped_promotions_to_database(scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage,
scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
def met_scrape_prepaid_promotins(soup, url, device_name, device_storage):
# make object
scraped_promotion = ScrapedPromotion()
# set variables already determined
scraped_promotion.provider = 'metropcs'
scraped_promotion.device_name = device_name
scraped_promotion.device_storage = device_storage
scraped_promotion.url = url
# make empty list of promotions
promotions = []
# crossed out price
try:
crossed_out_price = soup.find('span', class_='normal-price')
promotions.append([
'crossed out price',
crossed_out_price.text.strip().replace('\n', '').replace(
' ', '.')
])
except AttributeError:
crossed_out_price = ''
# make object for each promo text instance
for promo_instance in promotions:
scraped_promotion.promo_location = promo_instance[0]
scraped_promotion.promo_text = promo_instance[1]
# time variables
scraped_promotion.date = datetime.date.today()
scraped_promotion.time = datetime.datetime.now().time()
# add to database
add_scraped_promotions_to_database(
scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
def xfi_scrape_prepaid_promotins(url, device_name, device_storage,
description):
# make object
scraped_promotion = ScrapedPromotion()
# set variables already determined
scraped_promotion.provider = 'xfinity'
scraped_promotion.device_name = device_name
scraped_promotion.device_storage = device_storage
scraped_promotion.url = url
scraped_promotion.promo_text = description
scraped_promotion.promo_location = 'description'
# time variables
scraped_promotion.date = datetime.date.today()
scraped_promotion.time = datetime.datetime.now().time()
# add to database
add_scraped_promotions_to_database(
scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
def att_scrape_postpaid_promotions(soup, url, device_name, device_storage):
# make object
scraped_promotion = ScrapedPromotion()
# set variables already determined
scraped_promotion.provider = 'att'
scraped_promotion.device_name = device_name
scraped_promotion.device_storage = device_storage
scraped_promotion.url = url
# make empty list of promotions
promotions = []
# upper banner text
for span in soup.findAll("span", class_="offerTxt"):
if span.text.strip() != '':
upper_banner_text = span.text.strip()
promotions.append(['upper banner', upper_banner_text])
# lower banner text
for div in soup.findAll("div", class_="ds2MarketingMessageTextStyle"):
promotions.append(['lower banner', div.text])
# make object for each promo text instance
for promo_instance in promotions:
scraped_promotion.promo_location = promo_instance[0]
scraped_promotion.promo_text = promo_instance[1]
# hardcoded variables
scraped_promotion.date = datetime.date.today()
scraped_promotion.time = datetime.datetime.now().time()
add_scraped_promotions_to_database(
scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
def cri_scrape_prepaid_promotions(driver, url, device_name, device_storage):
# make object
scraped_promotion = ScrapedPromotion()
# set variables already determined
scraped_promotion.provider = 'cricket'
scraped_promotion.device_name = device_name
scraped_promotion.device_storage = device_storage
scraped_promotion.url = url
# make empty list of promotions
promotions = []
# sale price
try:
sale_price = driver.find_element_by_xpath(
'//*[@id="pricingWrapper"]/div[1]/div[1]')
promotions.append([
'sale price',
sale_price.text.strip().replace('\n', '').replace(
' ', '')
])
except NoSuchElementException:
sale_price = ''
# make object for each promo text instance
for promo_instance in promotions:
scraped_promotion.promo_location = promo_instance[0]
scraped_promotion.promo_text = promo_instance[1]
scraped_promotion.date = datetime.date.today()
scraped_promotion.time = datetime.datetime.now().time()
add_scraped_promotions_to_database(
scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
def ver_scrape_postpaid_promotions(soup, driver, url, device_name, device_storage):
# make object
scraped_promotion = ScrapedPromotion()
# set variables already determined
scraped_promotion.provider = 'verizon'
scraped_promotion.device_name = device_name
scraped_promotion.device_storage = device_storage
scraped_promotion.url = url
# make empty list of promotions
promotions = []
# alternate way to get banner text
upper_banner_text_2 = driver.find_element_by_class_name('clearfix')
if upper_banner_text_2.text.strip() != '':
promotions.append(['upper banner', upper_banner_text_2.text.strip()])
# crossed out price
pricing_options = soup.findAll('div', class_='pad8 noRightPad')
for div in pricing_options:
if 'was' in div.text:
promotions.append(['crossed out price', div.text.replace('2-Year Contract', ' 2-Year Contract').replace('24 Monthly Payments',' 24 Monthly Payments').replace('was ', ' was')])
# each payment option has its own banners
for option in range(1, len(pricing_options) + 1):
option_button = driver.find_element_by_xpath('//*[@id="tile_container"]/div[1]/div[3]/div[1]/div/div[2]/div/div/div[1]/div/div[' + str(option) + ']/div/div/div')
# PAYMENT LEVEL
# click on different payment options to show different promos
# if popup is there, remove it before clicking
try:
option_button.click()
except WebDriverException:
driver.find_element_by_class_name('fsrCloseBtn').click()
option_button.click()
time.sleep(2)
html = driver.page_source
soup = BeautifulSoup(html, "html.parser")
# promotion text above device icon
try:
banner_above_icon = soup.find('div', class_='offersPad fontSize_12 lineHeight8')
promotions.append(['banner above device icon', banner_above_icon.text.replace('Special Offer', '').replace('See the details', '').replace('\n', '')])
except AttributeError:
banner_above_icon = ''
# banner under price
below_price_banner = soup.find('div', class_='row padTop6 noSideMargin priceLabel').text
if below_price_banner != 'Retail Price' and below_price_banner != 'Early Termination Fee: $175 (2-Year Contracts)':
promotions.append(['banner below price', below_price_banner])
# make object for each promo text instance
for promo_instance in promotions:
scraped_promotion.promo_location = promo_instance[0]
scraped_promotion.promo_text = promo_instance[1]
# hardcoded variables
scraped_promotion.date = datetime.date.today()
scraped_promotion.time = datetime.datetime.now().time()
# add to database
add_scraped_promotions_to_database(scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
def cri_scrape_deals_page():
# headless Chrome
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1920x1080")
chrome_driver = os.getcwd() + "\\chromedriver.exe"
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path=chrome_driver)
driver.implicitly_wait(5)
# go to website
driver.get('https://www.cricketwireless.com/current-phone-and-plan-deals')
time.sleep(3)
html = driver.page_source
soup = BeautifulSoup(html, "html.parser")
# make object
scraped_promotion = ScrapedPromotion()
# set hardcoded variables
scraped_promotion.provider = 'cricket'
scraped_promotion.date = datetime.date.today()
scraped_promotion.time = datetime.datetime.now().time()
scraped_promotion.promo_location = 'deals page'
scraped_promotion.device_name = 'N/A'
scraped_promotion.device_storage = '0'
scraped_promotion.url = driver.current_url
# get big green promo banner
for div in soup.findAll('div', class_='hero-promo hover-item'):
deals_page_promo = format_promo_text(div.text)
scraped_promotion.promo_text = deals_page_promo
print(scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage,
scraped_promotion.promo_location, scraped_promotion.promo_text,
scraped_promotion.url, scraped_promotion.date,
scraped_promotion.time)
add_scraped_promotions_to_database(
scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
# get other main promotions
for div1 in soup.findAll('div', class_='promo-content-wrapper'):
deals_page_promo = format_promo_text(div1.text)
scraped_promotion.promo_text = deals_page_promo
print(scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage,
scraped_promotion.promo_location, scraped_promotion.promo_text,
scraped_promotion.url, scraped_promotion.date,
scraped_promotion.time)
add_scraped_promotions_to_database(
scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
# get promos at the bottom of the page
for div2 in soup.findAll('div', class_='col-xs-9'):
deals_page_promo = format_promo_text(div2.text)
scraped_promotion.promo_text = deals_page_promo
print(scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage,
scraped_promotion.promo_location, scraped_promotion.promo_text,
scraped_promotion.url, scraped_promotion.date,
scraped_promotion.time)
add_scraped_promotions_to_database(
scraped_promotion.provider, scraped_promotion.device_name,
scraped_promotion.device_storage, scraped_promotion.promo_location,
scraped_promotion.promo_text, scraped_promotion.url,
scraped_promotion.date, scraped_promotion.time)
driver.close()