def get_home_html(url):
try:
zl.navigate_to_website(driver, url)
output.append(driver.page_source)
print("Scraped: " + url)
except:
print("Invalid url: " + url)
# as many house listings as possible. If you want to use search terms other
# than zip codes, simply skip running zipcodes_list() function below, and add
# a line of code to manually assign values to object st, for example:
# st = ['Chicago', 'New Haven, CT', '77005', 'Jacksonville, FL']
# Keep in mind that, for each search term, the number of listings scraped is
# capped at 520, so in using a search term like "Chicago" the scraper would
# end up missing most of the results.
# Param st_items can be either a list of zipcode strings, or a single zipcode
# string.
st = zl.zipcodes_list(st_items=["111"])
# Initialize the webdriver.
driver = zl.init_driver("chromedriver.exe")
# Go to www.zillow.com/homes
zl.navigate_to_website(driver, "http://www.zillow.com/homes")
# Click the "buy" button.
zl.click_buy_button(driver)
# Create 11 variables from the scrapped HTML data.
# These variables will make up the final output dataframe.
df = pd.DataFrame({
'address': [],
'bathrooms': [],
'bedrooms': [],
'city': [],
'days_on_zillow': [],
'price': [],
'sale_type': [],
'state': [],
# than zip codes, simply skip running zipcodes_list() function below, and add
# a line of code to manually assign values to object st, for example:
# st = ["Chicago", "New Haven, CT", "77005", "Jacksonville, FL"]
# Keep in mind that, for each search term, the number of listings scraped is
# capped at 520, so in using a search term like "Chicago" the scraper would
# end up missing most of the results.
# Param st_items can be either a list of zipcode strings, or a single zipcode
# string.
#st = ["85201"]
# Initialize the webdriver.
driver = zl.init_driver("C:\webdrivers\chromedriver.exe")
# Go to www.zillow.com/homes
zl.navigate_to_website(driver, "http://www.google.com")
print("Enter a zip code")
time.sleep(30.091)
# Initialize list obj that will house all scraped data.
output_data = []
raw_data = zl.get_html(driver)
print("%s pages of listings found" % str(len(raw_data)))
# Take the extracted HTML and split it up by individual home listings.
listings = zl.get_listings(raw_data)
print("%s home listings scraped\n***" % str(len(listings)))
# For each home listing, extract the 11 variables that will populate that
import pandas as pd
import zillow_functions as zl
from bs4 import BeautifulSoup
# Enter zipcode term here
st = zl.zipcodes_list(st_items=["91320"])
nm = str(st[0])
# Initialize the webdriver.
# Use the location of the chromedriver file in your machine
# in a PC would be somethign like
# driver = zl.init_driver("C:/Users/username/chromedriver.exe")
driver = zl.init_driver("/Users/bjaimes/Desktop/nu/chromedriver")
# Go to https://www.zillow.com/homes/recently_sold
zl.navigate_to_website(driver, "https://www.zillow.com/homes/recently_sold")
# Create 10 variables from the scrapped HTML data.
# These variables will make up the final output dataframe.
df = pd.DataFrame({
'address': [],
'city': [],
'state': [],
'zip': [],
'sqft': [],
'price_sqft': [],
'calculated_price': [],
'bathrooms': [],
'bedrooms': [],
'zpid': [],
'latitude': [],
# Keep in mind that, for each search term, the number of listings scraped is
# capped at 520, so in using a search term like "Chicago" the scraper would
# end up missing most of the results.
# Param st_items can be either a list of zipcode strings, or a single zipcode
# string.
#st = zl.zipcodes_list(st_items = ["48103"])
st = ["Tuscon, AZ"]
# Initialize the webdriver.
driver = zl.init_driver("/Applications/chromedriver")
#os.environ["webdriver.chrome.driver"] = driver
# Go to www.zillow.com/homes/rs
#zl.navigate_to_website(driver, "http://www.zillow.com/homes/recently_sold/house_type/")
zl.navigate_to_website(driver, 'https://www.zillow.com/homes/tuscon-az_rb/')
# Click the "buy" button.
#zl.click_buy_button(driver)
# Get total number of search terms.
num_search_terms = len(st)
# Initialize list obj that will house all scraped data.
output_data = []
# Start the scraping.
for idx, term in enumerate(st):
# Enter search term and execute search.
'''if zl.enter_search_term(driver, term):
print("Entering search term %s of %s" %
def search(event):
# Create list of search terms.
# Function zipcodes_list() creates a list of US zip codes that will be
# passed to the scraper. For example, st = zipcodes_list(['10', '11', '606'])
# will yield every US zip code that begins with '10', begins with "11", or
# begins with "606" as a single list.
# I recommend using zip codes, as they seem to be the best option for catching
# as many house listings as possible. If you want to use search terms other
# than zip codes, simply skip running zipcodes_list() function below, and add
# a line of code to manually assign values to object st, for example:
# st = ['Chicago', 'New Haven, CT', '77005', 'Jacksonville, FL']
# Keep in mind that, for each search term, the number of listings scraped is
# capped at 520, so in using a search term like "Chicago" the scraper would
# end up missing most of the results.
# Param st_items can be either a list of zipcode strings, or a single zipcode
# string.
global list_of_zipcodes
if len(list_of_zipcodes) == 0:
print("No inputs given")
zipcodes_label['text'] = "No Inputs Given"
return
st = zl.zipcodes_list(st_items=list(list_of_zipcodes))
# Initialize the webdriver.
driver = zl.init_driver(
"/Users/jasontu/Projects/Real_Estate_Aggregator/Zillow/chromedriver")
# Go to www.zillow.com/homes
zl.navigate_to_website(driver, "http://www.zillow.com/homes")
# Click the "buy" button.
zl.click_buy_button(driver)
# Create 11 variables from the scrapped HTML data.
# These variables will make up the final output dataframe.
df = pd.DataFrame({
'address': [],
'bathrooms': [],
'bedrooms': [],
'city': [],
'days_on_zillow': [],
'price': [],
'sale_type': [],
'state': [],
'sqft': [],
'url': [],
'zip': []
})
# Get total number of search terms.
num_search_terms = len(st)
# Start the scraping.
for k in range(num_search_terms):
# Define search term (must be str object).
search_term = st[k]
# Enter search term and execute search.
if zl.enter_search_term(driver, search_term):
print("Entering search term number " + str(k + 1) + ": '" +
search_term + "' " + " out of " + str(num_search_terms))
else:
print("Search term " + str(k + 1) + ": '" + search_term + "' " +
" failed, moving onto next search term\n***")
continue
# Check to see if any results were returned from the search.
# If there were none, move onto the next search.
if zl.results_test(driver):
print("Search " + str(search_term) +
" returned zero results. Moving onto the next search\n***")
continue
# Pull the html for each page of search results. Zillow caps results at
# 20 pages, each page can contain 26 home listings, thus the cap on home
# listings per search is 520.
raw_data = zl.get_html(driver)
print(str(len(raw_data)) + " pages of listings found")
# Take the extracted HTML and split it up by individual home listings.
listings = zl.get_listings(raw_data)
# For each home listing, extract the 11 variables that will populate that
# specific observation within the output dataframe.
for n in range(len(listings)):
soup = BeautifulSoup(listings[n], "lxml")
new_obs = []
# List that contains number of beds, baths, and total sqft (and
# sometimes price as well).
card_info = zl.get_card_info(soup)
# Street Address
new_obs.append(zl.get_street_address(soup))
# Bathrooms
new_obs.append(zl.get_bathrooms(card_info))
# Bedrooms
new_obs.append(zl.get_bedrooms(card_info))
# City
new_obs.append(zl.get_city(soup))
# Days on the Market/Zillow
new_obs.append(zl.get_days_on_market(soup))
# Price
new_obs.append(zl.get_price(soup, card_info))
# Sale Type (House for Sale, New Construction, Foreclosure, etc.)
new_obs.append(zl.get_sale_type(soup))
# Sqft
new_obs.append(zl.get_sqft(card_info))
# State
new_obs.append(zl.get_state(soup))
# URL for each house listing
new_obs.append(zl.get_url(soup))
# Zipcode
new_obs.append(zl.get_zipcode(soup))
# Append new_obs to df as a new observation
if len(new_obs) == len(df.columns):
df.loc[len(df.index)] = new_obs
# Close the webdriver connection.
zl.close_connection(driver)
# Write df to CSV.
columns = [
'address', 'city', 'state', 'zip', 'price', 'sqft', 'bedrooms',
'bathrooms', 'days_on_zillow', 'sale_type', 'url'
]
df = df[columns]
dt = time.strftime("%Y-%m-%d") + "_" + time.strftime("%H%M%S")
file_name = str(dt) + ".csv"
df.to_csv(file_name, index=False)
zipcodes_label[
'text'] = "Scraping Complete. Review the following CSV file: \n" + str(
dt) + ".csv"
return
import time
import datetime
import pandas as pd
import zillow_functions as zl
import chromedriver_binary
url = 'https://www.zillow.com/homes/for_sale/Detroit-MI/pmf,pf_pt/17762_rid/LAND-CONTRACT_att/globalrelevanceex_sort/42.858853,-82.409821,41.721105,-84.38736_rect/8_zm/'
output_data = []
# Initialize the webdriver.
driver = zl.init_driver(chromedriver_binary.chromedriver_filename)
# Go to www.zillow.com/homes
zl.navigate_to_website(driver, url)
raw_data = zl.get_html(driver)
print("%s pages of listings found" % str(len(raw_data)))
# Take the extracted HTML and split it up by individual home listings.
listings = zl.get_listings(raw_data)
print("%s home listings scraped\n***" % str(len(listings)))
# For each home listing, extract the 11 variables that will populate that
# specific observation within the output dataframe.
for home in listings:
new_obs = []
parser = zl.html_parser(home)
# Street Address
new_obs.append(parser.get_street_address())
# capped at 520, so in using a search term like "Chicago" the scraper would
# end up missing most of the results.
# Param st_items can be either a list of zipcode strings, or a single zipcode
# string.
# st = zl.zipcodes_list(st_items = ["100", "770"])
# get zipcodes by location and radius
st = zl.zipcodes_list((34.016939, -118.185359, 20))
# Initialize the webdriver.
driver = zl.init_driver("C:/Users/username/chromedriver.exe")
# Go to www.zillow.com/homes
# zl.navigate_to_website(driver, "http://www.zillow.com/homes")
zl.navigate_to_website(driver, "https://www.zillow.com/los-angeles-ca/duplex/")
# Click the "buy" button.
zl.click_buy_button(driver)
# Create 11 variables from the scrapped HTML data.
# These variables will make up the final output dataframe.
df = pd.DataFrame({
'address': [],
'bathrooms': [],
'bedrooms': [],
'city': [],
'days_on_zillow': [],
'price': [],
'sale_type': [],
'state': [],
