def adapt_data_for_output(self, changed_listings, rented_listings,
new_listings):
excel_file_handler = ExcelFileHandler.ExcelFileHandler()
excel_file_handler.setup_generic_excel_file()
row_offset = 0
excel_file_handler.write_to_excel_file(row_offset, 0,
"Changed listings:")
row_offset += 2
for i in range(0, len(changed_listings)):
for j, k in enumerate(changed_listings[i]):
excel_file_handler.write_to_excel_file(i + row_offset, j, k)
row_offset += len(changed_listings) + 1
excel_file_handler.write_to_excel_file(row_offset, 0,
"Rented listings:")
row_offset += 2
for i in range(0, len(rented_listings)):
excel_file_handler.write_to_excel_file(
i + row_offset, 0,
rented_listings[i][0] + " - " + rented_listings[i][1])
row_offset += len(rented_listings) + 1
excel_file_handler.write_to_excel_file(row_offset, 0, "New listings:")
row_offset += 2
for i in range(0, len(new_listings)):
for j, k in enumerate(new_listings[i]):
excel_file_handler.write_to_excel_file(i + row_offset, j, k)
row_offset += len(new_listings) + 1
excel_file_handler.finalize_spreadsheet(row_offset + 2,
self.total_listings,
"Metronest")
def get_apartment_links(self, driver):
finalized_info_list = []
list_of_apartment_links = []
available_floorplans = self.does_floorplan_have_apartments_available(
driver)
total_floorplans_to_check = len(available_floorplans)
for i in range(0, total_floorplans_to_check):
select_floor_plan_element = driver.find_element_by_id("step2")
available_floorplans = self.does_floorplan_have_apartments_available(
driver)
list_of_apartment_links.append(
self.find_all_apartments_from_floorplans(
available_floorplans[i], driver))
single_entry_dict = self.scrape_apartment_link(
list_of_apartment_links[i][0], driver)
excel_adapter = ExcelFileHandler.ExcelFileAdapter(
single_entry_dict) #, "1133 Manhattan")
finalized_info_list.append(excel_adapter.adapt_info_for_output())
self.click_on_an_element(select_floor_plan_element)
return finalized_info_list
def scrape_data(self, driver, progress_text):
raw_apartment_list = []
finalized_info_list = []
apartment_details = []
driver.maximize_window()
self.deal_with_accepting_cookies(driver)
progress_text("Waiting for website to load...")
time.sleep(15)
floorplans_links = [
data for data in driver.find_elements_by_tag_name("div") if
data.get_attribute("class") == "floorplan-detail-toggle fp-col-6"
]
for link in floorplans_links:
self.click_on_an_element(link, driver)
close_button = driver.find_element_by_class_name("close-detail")
for data in driver.find_elements_by_tag_name('ul'):
if data.get_attribute('class') == 'fp-detail-unit':
apartment_details = data.text.split('\n')
bed_and_bath = [
data.text
for data in driver.find_elements_by_tag_name("span")
if data.get_attribute("class") == "bed-count"
or data.get_attribute("class") == "bath-count"
]
apartment_details[3] = bed_and_bath[-2]
apartment_details.append(bed_and_bath[-1])
polished_data = self.polish_data(apartment_details)
dict_apartment_info = self.format_data_into_dict(
polished_data)
excel_handler = ExcelFileHandler.ExcelFileAdapter(
dict_apartment_info)
finalized_info_list.append(
excel_handler.adapt_info_for_output())
self.click_on_an_element(close_button, driver)
return finalized_info_list
import LinkScraper
import DataComparison
import ElevenThirteenManhattan
import TheEugene
import CitySkyline
import Bettina
import R9300
import Solil
import Eberhart
import Metronest
import time
import sys
fileHandler = FileHandler.FileHandler()
logHandler = LogHandler.LogHandler()
excelFileHandler = ExcelFileHandler.ExcelFileHandler()
htmlFileScraper = HTMLFileScraper.HTMLFileScraper()
linkScraper = LinkScraper.LinkScraper()
ETManhattan = ElevenThirteenManhattan.ElevenThirteenManhattan()
theEugene = TheEugene.TheEugene()
class GUI:
progress_text = StringVar
links_remaining_text = StringVar
coding = StringVar
file_folder_import_switch = StringVar
owner_name = 'owner_name'
html_file = 'html_file'
def __init__(self):
self.excel_file_handler = ExcelFileHandler.ExcelFileHandler()