def scrap_result_row(self, div):
try:
div = div.find('div', class_='prodInfo')
sub_div = div.find('div', class_='prodName')
a = sub_div.find('a')
print '[BedBathAndBeyondScraper] :: title: ', a.text.strip()
div = div.find('div', class_='prodPrice')
sub_div = div.find('div', class_='priceOfProduct')
sub = sub_div.find('div', class_='isPrice')
print '[BedBathAndBeyondScraper] :: price: ', sub.text.strip()
self.mdb.bedbathandbeyond_scraper_data(a.text.strip(),
sub.text.strip())
fname = 'data_bed_bath_and_beyond.csv'
msg = "%s, %s," % (a.text.strip(), sub.text.strip())
print "[BedBathAndBeyondScraper] :: scrap_result_row() :: " \
"msg:", msg
scraper_csv_write(fname, msg)
except Exception as exp:
print '[BedBathAndBeyondScraper] :: scrap_result_row() :: ' \
'Got exception : %s' % exp
print(traceback.format_exc())
def scrap_result_row(self, div):
try:
# title
title = div.find('span', class_='company').text.strip()
print "[IndeedScrapper] :: title: %s" % title
# location
span = div.find('span', class_='location')
location = span.text.strip()
print "[IndeedScrapper] :: location: %s" % location
# salary
sal = ''
span = div.find('span', class_='no-wrap')
if span:
sal = span.text.strip()
print "[IndeedScrapper] :: salary: %s" % sal
# summary
span = div.find('span', class_='summary')
summary = span.text.strip()
print "[IndeedScrapper] :: summery: %s" % summary
self.mdb.indeed_scraper_data(title, location, sal, summary)
fname = 'data_indeed.csv'
msg = "%s, %s, %s, %s," % (title, location, sal, summary)
print "[IndeedScrapper] :: scrap_result_row() :: msg:", msg
scraper_csv_write(fname, msg)
except Exception as exp:
print '[IndeedScrapper] :: scrap_result_row() :: ' \
'Got exception : %s' % exp
def scrap_result_row(self, div):
try:
c_wiz = div.find('c-wiz', class_='M1Uqc kWyHVd')
headlines = c_wiz.find('a', class_='nuEeue hzdq5d ME7ew')\
.text.strip()
print '[GoogleNewsScraper] :: HeadLines: ', headlines
div = div.find('div', class_='alVsqf')
sub = div.find('div', class_='jJzAOb')
c_wiz = sub.find('c-wiz', class_='M1Uqc MLSuAf')
a = c_wiz.find('a', class_='nuEeue hzdq5d ME7ew').text.strip()
print '[GoogleNewsScraper] :: SubheadLines: ', a
# save in data base
self.mdb.google_news_data(headlines, a)
fname = 'data_google_news.csv'
msg = "%s, %s" % (headlines, a)
print "[GoogleNewsScraper] :: scrap_result_row() :: msg:", msg
scraper_csv_write(fname, msg)
except Exception as exp:
print '[GoogleNewsScraper] :: scrap_result_row() :: ' \
'Got exception : %s' % exp
print(traceback.format_exc())
def scrap_result_row(self, div):
try:
# # name
# name = div.find('div', class_='pod-plp__description js-
# podclick-analytics')
# a = name.find('a').strip()
# print '[HomeDepotScraper] :: name: ', a
# model
model = div.find('div', class_='pod-plp__model').text.strip()
print '[HomeDepotScraper] :: model: ', model
# price
price = div.find('div', class_='price').text.strip()
print '[HomeDepotScraper] :: price: ', price
# stock
stock = div.find('div', class_='pod-plp__shipping-message__'
'wrapper-boss-bopis').text.strip()
print '[HomeDepotScraper] :: stock: ', stock
self.mdb.homedepot_data(model, price, stock)
fname = 'data_home_depot.csv'
msg = "%s, %s, %s," % (model, price, stock)
print "[HomeDepotScraper] :: scrap_result_row() :: msg:", msg
scraper_csv_write(fname, msg)
except Exception as exp:
print '[HomeDepotScraper] :: scrap_result_row() :: ' \
'Got exception : %s' % exp
def scrap_result_row(self, div):
try:
h2 = div.find('h2', class_='n')
title = div.find('a', class_='business-name').text.strip()
print "[YellowPagesScraper] :: title: %s" % title
rating_count = 0
span = div.find('span', class_='count')
if span:
span = span.text.strip()
rating_count = span
print "[YellowPagesScraper] :: rating_count: %s" % rating_count
p = div.find('p', class_='adr')
address = p.text
print "[YellowPagesScraper] :: address: %s" % address
phone = ''
li = div.find('li', class_='phone primary')
if li:
phone = li.text.strip()
print "[YellowPagesScraper] :: phone: %s" % phone
else:
print "[YellowPagesScraper] :: phone: %s" % li
categories = ''
cat_div = div.find('div', class_='categories')
if cat_div:
categories = cat_div.text.strip()
print "[YellowPagesScraper] :: categories: %s" % categories
else:
print "[YellowPagesScraper] :: categories: %s" % cat_div
self.mdb.yellowpages_scraper_data(title, rating_count, address,
phone, categories)
fname = 'data_yellow_pages.csv'
msg = "%s, %s, %s, %s, %s" % (title, rating_count, address, phone,
categories)
print "[YellowPagesScraper] :: scrap_result_row() :: msg:", msg
scraper_csv_write(fname, msg)
except Exception as exp:
print '[YellowPagesScraper] :: scrap_result_row() :: ' \
'Got exception: %s' % exp
print(traceback.format_exc())
def scrap_result_row(self, div):
try:
Product_div = div.find('div', class_='col col-7-12')
title = Product_div.find('div', class_='_3wU53n').text.strip()
print('[FlipkartScraper] :: title . . . . ..:', title)
# title_description = div.find('div', class_='OiPjke').text.strip()
# print'[FlipkartScraper] :: title_description: ', title_description
rating = div.find('div', class_='niH0FQ')
sub_rating = rating.find('span', class_='_38sUEc').text.strip()
print('[FlipkartScraper] :: rating . . . . .:', sub_rating)
specifications_div = div.find('div', class_='_3ULzGw')
specifications = specifications_div.find(
'ul', class_='vFw0gD').text.strip()
print('[FlipkartScraper] :: specifications .: ', specifications)
product_price = div.find('div', class_='_6BWGkk')
div_price = product_price.find('div', class_='_1uv9Cb')
price = div_price.find('div',
class_='_1vC4OE _2rQ-NK').text.strip()
print('[FlipkartScraper] :: price . . . . . :',
price.encode("utf-8"))
self.mdb.flipkart_scrapper_data(title, sub_rating, specifications,
price)
fname = 'data_flipkart.csv'
msg = "%s, %s, %s, %s," % (title, sub_rating, specifications,
price.encode("utf-8"))
print("[FlipkartScraper] :: scrap_result_row() :: msg:", msg)
scraper_csv_write(fname, msg)
except Exception as exp:
print ('[FlipkartScraper] :: scrap_result_row() :: ' \
'Got exception: %s' % exp)
print(traceback.format_exc())
def scrap_result_row(self, div):
try:
div = div.find('div', class_='product-info')
sub_div = div.find('div', class_='product-price-wrapper')
price = sub_div.find('div', class_='product-price-container')\
.text.strip()
print '[OverStockScraper] :: price: ', price
title = div.find('div', class_='product-title').text.strip()
print '[OverStockScraper] :: title: ', title
rating = div.find('div', class_='product-footer')
print '[OverStockScraper] :: rating: ', rating
self.mdb.overstock_scraper_data(price, title, rating)
fname = 'data_over_stock.csv'
msg = "%s, %s, %s," % (price, title, rating)
print "[OverStockScraper] :: scrap_result_row() :: msg:", msg
scraper_csv_write(fname, msg)
except Exception as exp:
print '[OverStockScraper] :: scrap_result_row() :: ' \
'Got exception: %s' % exp
print(traceback.format_exc())
def scrap_row_yelp(self, li):
try:
h3 = li.find('h3', class_='search-result-title')
# Getting title
title = ''
spans = h3.find_all('span')
i = 0
for span in spans:
i += 1
if i == 2:
title = span.text.strip()
print "[YelpScraper] :: title: %s" % title
# Getting reviews count
reviews_count = 0
span = li.find('span', class_='review-count rating-qualifier')
text = span.text
lst = text.split()
reviews_count = int(lst[0])
print "[YelpScraper] :: reviews count: %d" % reviews_count
# Getting services
services = []
span = li.find('span', class_='category-str-list')
text = span.text
lst = text.split(',')
services = [itm.strip() for itm in lst]
print "[YelpScraper] :: services: %s" % services
# Getting address
address = li.find('address').text.strip()
print "[YelpScraper] :: address: %s" % address
# Getting phone
phone = li.find('span', class_='biz-phone').text.strip()
print "[YelpScraper] :: phone: %s" % phone
# Getting snippet
p = li.find('p', class_='snippet').text.strip()
lst = p.split('read more')
snippet = lst[0].strip()
print "[YelpScraper] :: snippet: %s" % snippet
self.mdb.yelp_scraper_data(title, reviews_count, services, address,
phone, snippet)
fname = 'data_yelp.csv'
msg = "%s, %s, %s, %s, %s, %s" % (title, reviews_count, services,
address, phone, snippet)
print "[IndeedScrapper] :: scrap_result_row() :: CSV file Msg: ", msg
scraper_csv_write(fname, msg)
except Exception as exp:
print '[YelpScraper] :: scrap_row_yelp() :: Got exception: %s' % exp
print(traceback.format_exc())
def scrap_result_row(self, div):
try:
#get title of company
h2 = div.find('h2', class_='n')
title = h2.find('a', class_='business-name').text.strip()
print ("[YellowPagesScraper] :: title: %s" % title)
#get rating_count of company
rating_count = 0
span = div.find('span', class_='count')
if span:
span = span.text.strip()
rating_count = span
print ("[YellowPagesScraper] :: rating_count: %s" % rating_count)
#get the address of the company
p = div.find('p', class_='adr')
street_address = p.find('div',class_='street-address')
if street_address:
str_adrs = street_address.text.strip()
locality = p.find('div',class_='locality').text.strip()
address = str_adrs + locality
print ("[YellowPagesScraper] :: address: %s" % address)
address=''
else:
#get the contact Number of Company
phone = ''
li = div.find('div', class_='phones phone primary')
if li:
phone = li.text.strip()
print ("[YellowPagesScraper] :: phone: %s" % phone)
else:
print ("[YellowPagesScraper] :: phone: %s" % li)
#get the categories of company
categories = ''
cat_div = div.find('div', class_='categories')
category = cat_div.find('a')
if category:
categories = cat_div.text.strip()
print ("[YellowPagesScraper] :: categories: %s" % categories)
else:
print ("[YellowPagesScraper] :: categories: %s" % cat_div)
#get the website of the company
div22=div.find('div',class_='info-section info-primary')
webpage_link = div22.find('a', {'class': 'track-visit-website'})
if webpage_link:
website_link = (webpage_link)['href']
print('[YellowPagesScraper] :: webpage_link: %s' % website_link)
else:
website_link=''
self.mdb.yellowpages_scrapper_data(title, rating_count, address, phone, categories,website_link)
fname = 'data_yellow_pages.csv'
msg = "%s, %s, %s, %s, %s, %s" % (title, rating_count, address, phone, categories, website_link)
#print ("[YellowPagesScraper] :: scrap_result_row() :: msg:", msg)
scraper_csv_write(fname, msg)
# except:
# pass
except Exception as exp:
print ('[YellowPagesScraper] :: scrap_result_row() :: ' \
'Got exception: %s' % exp)
print(traceback.format_exc())