def _default_scraping_routine(self, page_source):
##
# ...
#
# @param {string} page_source - ...
#
# @return {list}
#
hotel_results = []
soup = bs4.BeautifulSoup(page_source, 'html.parser')
# search_info_span = soup.select("#breadcrumb div:nth-of-type(5) span")
# search_target_adults = search_info_span[0].contents[1].string.encode("utf-8").strip()
# search_nights = search_info_span[0].contents[2].string.encode("utf-8").strip()
# search_dates = search_info_span[0].contents[3].string.encode("utf-8").strip()
# search_info = "{} {} {}".format(search_target_adults, search_nights, search_dates)
hotellist_items = soup.select("#resultsContainer .hotel.listing")
for div in hotellist_items:
hotelname = price_text = number_of_nights_text = rating_value = \
rating_unit = location = None
# Extract the name of the hotel.
strong_name = div.select(".hotelTitle > .hotelName")
if len(strong_name) > 0:
hotelname = strong_name[0].string.encode("utf-8").strip()
location_tag = div.select(".hotel-info .neighborhood")
if len(location_tag) > 0:
location = list(
location_tag[0].strings)[0].encode("utf-8").strip()
# Extract the number of stars.
span_stars = div.select(
"li.starRating strong.star-rating > span:nth-of-type(2)")
if len(span_stars) > 0:
star_classes = span_stars[0].get("class")
for star_class in star_classes:
if "icon-stars" in star_class:
rating_value = float(star_class[-3:].replace("-", "."))
rating_unit = "stars"
break
else:
rating_unit = None
rating_value = 0.0
# Extract the price of the hotel.
price_tag = div.select(".hotel-price .actualPrice")
if len(price_tag) > 0:
price_text = list(
price_tag[0].strings)[-1].encode("utf-8").strip()
# Extract the number of nights, the price stands for.
li_nights = div.select("li.avgPerNight.priceType")
if len(li_nights) > 0:
number_of_nights_text = list(
li_nights[0].strings)[-1].encode("utf-8").strip()
if hotelname is not None:
number_of_nights = 1
# split price and currency.
price, currency = \
CurrencyConverter.split_price_and_currency(price_text=price_text)
currency_code = \
CurrencyConverter.get_currency_code_of_sign(currency_sign=currency)
# get the normalized price from the api function
price_norm = CurrencyConverter.get_normalized_price(
price=price, currency_code=currency_code)
hotel_results.append({
"name":
hotelname,
"location":
location,
"price":
price,
"currency":
currency_code,
"price_norm":
price_norm,
"number_of_nights":
number_of_nights,
"rating_value":
rating_value,
"rating_unit":
rating_unit,
"access_time":
time.strftime("%d-%m-%Y %H:%M:%S", time.gmtime()),
"debug": {
"price_text": price_text,
"number_of_nights_text": number_of_nights_text,
},
})
return hotel_results
def _alternative_scraping_routine(self, page_source):
##
# ...
#
# @param {string} page_source - ...
#
# @return {list}
#
regex_price = re.compile("([0-9]+[ .])*[0-9]+")
hotel_results = []
soup = bs4.BeautifulSoup(page_source, 'html.parser')
# search_info_span = soup.select("#breadcrumb div:nth-of-type(5) span")
# search_target_adults = search_info_span[0].contents[1].string.encode("utf-8").strip()
# search_nights = search_info_span[0].contents[2].string.encode("utf-8").strip()
# search_dates = search_info_span[0].contents[3].string.encode("utf-8").strip()
# search_info = "{} {} {}".format(search_target_adults, search_nights, search_dates)
hotellist_items = soup.select(
".hotelSlimResultsModuleMod > div > div.hotel-result")
for div in hotellist_items:
hotelname = price_text = number_of_nights_text = None
a_name = div.select("h2.hotel-result-title > a")
if len(a_name) > 0:
hotelname = a_name[0].string.encode("utf-8").strip()
b_price = div.select(".primary-price strong")
if len(b_price) > 0:
price_text = b_price[0].string.encode("utf-8").strip()
# per night info
div_nights = div.select(".rate-choice-msg")
if len(div_nights) > 0:
number_of_nights_text = div_nights[0].string.encode(
"utf-8").strip()
if hotelname != None:
number_of_nights = 1
# split price and currency.
price, currency = \
CurrencyConverter.split_price_and_currency(price_text=price_text)
currency_code = \
CurrencyConverter.get_currency_code_of_sign(currency_sign=currency)
# get the normalized price from the api function
price_norm = CurrencyConverter.get_normalized_price(
price=price, currency_code=currency_code)
hotel_results.append({
"name":
hotelname,
"price":
price,
"currency":
currency_code,
"price_norm":
price_norm,
"number_of_nights":
number_of_nights,
"access_time":
time.strftime("%d-%m-%Y %H:%M:%S", time.gmtime()),
"debug": {
"price_text": price_text,
"number_of_nights_text": number_of_nights_text,
},
})
return hotel_results
def _scraping_routine(self, page_source):
orbitz_car_results = []
bsObj = bs4.BeautifulSoup(page_source, 'html.parser')
hotellist_items = bsObj.select("#search-results .listing-wrapper")
for div in hotellist_items:
price_daily_text = price_total_text = car_class = company_name = \
car_model = None
# Determine the daily price of the current car.
price_daily_element = div.find_all("div", {"class":"full-price"})
if len(price_daily_element) > 0:
price_daily_text = self._encode_and_strip(price_daily_element[0])
# Determine the total price of the current car.
price_total_element = div.find_all("div", {"class":"total"})
if len(price_total_element) > 0:
# TODO split "total" string from price.
price_total_text = self._encode_and_strip(price_total_element[0])
# Determine the class of the car.
car_class_element = div.select("div.fullName span")
if len(car_class_element) > 0:
car_class = self._encode_and_strip(car_class_element[0])
# Determine the company name of the current car.
company_name_element = div.select("div.vendor-image-box img")
if len(company_name_element) > 0:
# TODO Test whether the alt content is found or not.
company_name = self._encode_and_strip(company_name_element[0]["alt"])
# Determine the model of the current car.
car_model_element = div.select(".car-model")
if len(car_model_element) > 0:
car_model = self._encode_and_strip(car_model_element[0])
# Check if all information of the current car are available.
if price_daily_text is not None and price_total_text is not None and \
car_class is not None and company_name is not None and car_model is not None:
# Normalize the daily and total price for the current car.
#
# split price and currency of daily price.
price_daily, currency_daily = \
CurrencyConverter.split_price_and_currency(price_text=price_daily_text)
currency_code_daily = \
CurrencyConverter.get_currency_code_of_sign(currency_sign=currency_daily)
# get the normalized daily price from the api function
price_norm_daily = CurrencyConverter.get_normalized_price(
price=price_daily,
currency_code=currency_code_daily
)
# split price and currency of total price.
price_total, currency_total = \
CurrencyConverter.split_price_and_currency(price_text=price_total_text)
currency_code_total = \
CurrencyConverter.get_currency_code_of_sign(currency_sign=currency_total)
# get the normalized total price from the api function
price_norm_total = CurrencyConverter.get_normalized_price(
price=price_total,
currency_code=currency_code_total
)
# Store all the received information in the result list.
orbitz_car_results.append({
"company_name": company_name,
"car_model": car_model,
"car_class" : car_class,
"price_daily": price_daily,
"price_norm_daily" : price_norm_daily,
"price_total": price_total,
"price_norm_total" : price_norm_total,
"currency" : currency_code_daily,
"access_time": time.strftime("%d-%m-%Y %H:%M:%S", time.gmtime()),
},
)
return orbitz_car_results
def _scraping_routine(self, page_source):
##
# ...
#
# @param {string} page_source - ...
#
# @return {list}
#
regex_price_per_nights = re.compile("[0-9]+")
hotel_results = []
soup = bs4.BeautifulSoup(page_source, 'html.parser')
# Extract info about the search values. For debug.
div_search_info = soup.select(".dates-occupancy")
search_dates = div_search_info[0].select(".search-dates")[0].string.encode("utf-8").strip()
search_nights = div_search_info[0].select(".search-nights")[0].string.encode("utf-8").strip()
search_rooms = div_search_info[0].select(".search-rooms")[0].string.encode("utf-8").strip()
search_info = "{dates}, {nights}, {rooms}".format(
dates=search_dates, nights=search_nights, rooms=search_rooms)
# Start with the extraction of hotel information.
hotellist_items = soup.select("div#listings > ol.listings > li.hotel")
for div in hotellist_items:
hotelname = price = price_text = number_of_nights = \
number_of_nights_text = currency_code = price_norm = \
rating_value = rating_unit = location = None
# Extract the hotel name.
a_name = div.select("h3.p-name a")
if len(a_name) > 0:
hotelname = a_name[0].string.encode("utf-8").strip()
# Extract location.
address_element = div.select(".contact .p-adr")
if len(address_element) > 0:
location = "".join(list(address_element[0].strings)[:-1]).encode('utf-8').strip()
# Extract the stars rating.
span_star_rating = div.select("span.star-rating.widget-star-rating-overlay")
if len(span_star_rating) > 0:
rating_value = float(span_star_rating[0].get("data-star-rating"))
rating_unit = "stars"
else:
rating_value = 0.0
# try to extract the normal price.
b_price = div.select(".price b")
if len(b_price) > 0:
price_text = b_price[0].string.encode("utf-8").strip()
else:
# if price was reduced (red colored price)
ins_price = div.select(".price span.old-price-cont ins")
if len(ins_price) > 0:
price_text = ins_price[0].string.encode("utf-8").strip()
# determine the number of nights the extracted price stands for.
span_price_info = div.select(".price-breakdown > .price-info")
if len(span_price_info) > 0:
number_of_nights_text = span_price_info[0].string.encode("utf-8").strip()
match = regex_price_per_nights.search(number_of_nights_text)
if match != None:
number_of_nights = int(number_of_nights_text[match.start():match.end()])
else:
# Fallback, because if the price is per night, there is not
# always a 1 in the string.
number_of_nights = 1
# save hotel in results, if all information were extracted.
if hotelname != None and price_text != None and number_of_nights != None:
# split price and currency.
price, currency = \
CurrencyConverter.split_price_and_currency(price_text=price_text)
currency_code = \
CurrencyConverter.get_currency_code_of_sign(currency_sign=currency)
# get the normalized price from the api function
price_norm = CurrencyConverter.get_normalized_price(
price=price,
currency_code=currency_code
)
if price_norm is not None:
# calc price for one night
price_norm = round(price_norm / number_of_nights, 2)
hotel_results.append({
"name" : hotelname,
"location" : location,
"price" : price,
"currency" : currency_code,
"price_norm" : price_norm,
"number_of_nights" : number_of_nights,
"rating_value" : rating_value,
"rating_unit" : rating_unit,
"access_time" : time.strftime("%d-%m-%Y %H:%M:%S", time.gmtime()),
"debug" : {
"price_text" : price_text,
"number_of_nights_text" : number_of_nights_text,
"search_info" : search_info,
},
})
return hotel_results
def _touch_scraping_routine(self, page_source):
hotel_results = []
soup = bs4.BeautifulSoup(page_source, 'html.parser')
hotellist_items = soup.select("#resultList .listItem")
for div in hotellist_items:
hotelname = price = currency_code = price_norm = None
# Extract hotel name.
div_name = div.select(".hotelData > .labeled")
if len(div_name) > 0:
hotelname = div_name[0].string.encode("utf-8").strip()
# Extract rating information (stars)
stars_element = div.select(".hotelData .smaller .stars")
if len(stars_element) > 0:
stars_element_classes = stars_element[0].get("class")
# Search the sX class which indicates the number of stars.
for star_class in stars_element_classes:
if star_class[-1].isdigit():
rating_value = float(star_class[1:])
rating_unit = "stars"
else:
rating_value = 0.0
rating_unit = None
# Extract hotel price.
span_price = div.select(".hotelData > .priceInfo span.price")
if len(span_price) > 0:
price_text = span_price[0].string.encode("utf-8").strip()
# Extract the price and currency from the price string.
price, currency = \
CurrencyConverter.split_price_and_currency(price_text=price_text)
# Get the currency code for the extracted currency.
currency_code = \
CurrencyConverter.get_currency_code_of_sign(currency_sign=currency)
# get the normalized price from the api function
price_norm = CurrencyConverter.get_normalized_price(
price=price, currency_code=currency_code)
if hotelname != None and price != None and price_norm != None:
# this information is not available on the website.
# it seems to be always the price per night
number_of_nights = 1
hotel_results.append({
"name":
hotelname,
"price":
price,
"currency":
currency_code,
"price_norm":
price_norm,
"number_of_nights":
number_of_nights,
"rating_value":
rating_value,
"rating_unit":
rating_unit,
"access_time":
time.strftime("%d-%m-%Y %H:%M:%S", time.gmtime()),
})
return hotel_results