def store_property_data():
"""
Stores the zillow response data for a given address/zipcode.
"""
zkey = current_app.config['ZKEY']
if not request.get_json() or 'address' not in request.json or 'zipcode' not in request.json:
abort(400)
zillow_data = ZillowWrapper(zkey)
address = request.json['address']
zipcode = request.json['zipcode']
try:
response = zillow_data.get_deep_search_results(address, zipcode)
result = GetDeepSearchResults(response)
except ZillowError as err:
return jsonify({'error': err.message, 'estimate': 0})
# delete from db if already exists to prevent duplicates
# (property data my have changed so need to refresh entry)
pd = PropertyData.objects(zillow_id=result.zillow_id)
for doc in pd:
doc.delete()
pd = fill_property_document(result)
pd.save()
return jsonify({'zillow_id': result.zillow_id, 'error': ''})
def test_zillow_api_connect(self):
# create response from zillow and check for error code '0'
zillow_data = ZillowWrapper(self.ZILLOW_API_KEY)
zillow_search_response = zillow_data.get_deep_search_results(
self.address, self.zipcode)
assert \
zillow_search_response.find('message').find('code').text == '0'
def test_get_updated_property_details_results(self):
"""
"""
zillow_id = '48749425'
responses.add(
responses.GET,
'http://www.zillow.com/webservice/GetUpdatedPropertyDetails.htm',
body='<?xml version="1.0" encoding="utf-8"?><UpdatedPropertyDetails:updatedPropertyDetails xmlns:UpdatedPropertyDetails="http://www.zillow.com/static/xsd/UpdatedPropertyDetails.xsd" xsi:schemaLocation="http://www.zillow.com/static/xsd/UpdatedPropertyDetails.xsd http://www.zillowstatic.com/vstatic/34794f0/static/xsd/UpdatedPropertyDetails.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><request><zpid>48749425</zpid></request><message><text>Request successfully processed</text><code>0</code></message><response><zpid>48749425</zpid><pageViewCount><currentMonth>16125</currentMonth><total>16125</total></pageViewCount><address><street>2114 Bigelow Ave N</street><zipcode>98109</zipcode><city>Seattle</city><state>WA</state><latitude>47.637933</latitude><longitude>-122.347938</longitude></address><links><homeDetails>http://www.zillow.com/homedetails/2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/</homeDetails><photoGallery>http://www.zillow.com/homedetails/2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/#image=lightbox%3Dtrue</photoGallery><homeInfo>http://www.zillow.com/homedetails/2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/</homeInfo></links><images><count>1</count><image><url>http://photos3.zillowstatic.com/p_d/ISxb3qa8s1cwx01000000000.jpg</url></image></images><editedFacts><useCode>SingleFamily</useCode><bedrooms>4</bedrooms><bathrooms>3.0</bathrooms><finishedSqFt>3470</finishedSqFt><lotSizeSqFt>4680</lotSizeSqFt><yearBuilt>1924</yearBuilt><yearUpdated>2003</yearUpdated><numFloors>2</numFloors><basement>Finished</basement><roof>Composition</roof><view>Water, City, Mountain</view><parkingType>Off-street</parkingType><heatingSources>Gas</heatingSources><heatingSystem>Forced air</heatingSystem><rooms>Laundry room, Walk-in closet, Master bath, Office, Dining room, Family room, Breakfast nook</rooms></editedFacts><neighborhood>Queen Anne</neighborhood><schoolDistrict>Seattle</schoolDistrict><elementarySchool>John Hay</elementarySchool><middleSchool>McClure</middleSchool></response></UpdatedPropertyDetails:updatedPropertyDetails><!-- H:001 T:127ms S:990 R:Sat Sep 12 23:47:31 PDT 2015 B:4.0.19615-release_20150908-endor.2fa5797~candidate.358c83d -->',
content_type='application/xml',
status=200
)
zillow_data = ZillowWrapper(api_key=None)
updated_property_details_response = \
zillow_data.get_updated_property_details(zillow_id)
result = GetUpdatedPropertyDetails(updated_property_details_response)
assert result.zillow_id == '48749425'
assert result.home_type == 'SingleFamily'
assert result.home_detail_link == \
'http://www.zillow.com/homedetails/' + \
'2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/'
assert result.photo_gallery == \
'http://www.zillow.com/homedetails/' + \
'2114-Bigelow-Ave-N-Seattle-WA-98109/' + \
'48749425_zpid/#image=lightbox%3Dtrue'
assert result.home_info == \
'http://www.zillow.com/homedetails/' + \
'2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/'
assert result.year_built == '1924'
assert result.property_size == '4680'
assert result.home_size == '3470'
assert result.bathrooms == '3.0'
assert result.bedrooms == '4'
assert result.year_updated == '2003'
assert result.basement == 'Finished'
assert result.roof == 'Composition'
assert result.view == 'Water, City, Mountain'
assert result.heating_sources == 'Gas'
assert result.heating_system == 'Forced air'
assert result.rooms == \
'Laundry room, Walk-in closet, Master bath, Office,' + \
' Dining room, Family room, Breakfast nook'
assert result.neighborhood == 'Queen Anne'
assert result.school_district == 'Seattle'
lat = float(result.latitude)
assert lat - 0.01 <= 47.637933 <= lat + 0.01
lng = float(result.longitude)
assert lng - 0.01 <= -122.347938 <= lng + 0.01
assert result.floor_material is None
assert result.num_floors == '2'
assert result.parking_type == 'Off-street'
def test_deep_search_results(self):
"""
"""
address = '2114 Bigelow Ave Seattle, WA'
zipcode = '98109'
responses.add(
responses.GET,
'http://www.zillow.com/webservice/GetDeepSearchResults.htm',
body='<?xml version="1.0" encoding="utf-8"?><SearchResults:searchresults xsi:schemaLocation="http://www.zillow.com/static/xsd/SearchResults.xsd http://www.zillowstatic.com/vstatic/34794f0/static/xsd/SearchResults.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:SearchResults="http://www.zillow.com/static/xsd/SearchResults.xsd"><request><address>2114 Bigelow Ave Seattle, WA</address><citystatezip>98109</citystatezip></request><message><text>Request successfully processed</text><code>0</code></message><response><results><result><zpid>48749425</zpid><links><homedetails>http://www.zillow.com/homedetails/2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/</homedetails><graphsanddata>http://www.zillow.com/homedetails/2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/#charts-and-data</graphsanddata><mapthishome>http://www.zillow.com/homes/48749425_zpid/</mapthishome><comparables>http://www.zillow.com/homes/comps/48749425_zpid/</comparables></links><address><street>2114 Bigelow Ave N</street><zipcode>98109</zipcode><city>Seattle</city><state>WA</state><latitude>47.637933</latitude><longitude>-122.347938</longitude></address><FIPScounty>53033</FIPScounty><useCode>SingleFamily</useCode><taxAssessmentYear>2014</taxAssessmentYear><taxAssessment>1060000.0</taxAssessment><yearBuilt>1924</yearBuilt><lotSizeSqFt>4680</lotSizeSqFt><finishedSqFt>3470</finishedSqFt><bathrooms>3.0</bathrooms><bedrooms>4</bedrooms><lastSoldDate>11/26/2008</lastSoldDate><lastSoldPrice currency="USD">1025000</lastSoldPrice><zestimate><amount currency="USD">1419804</amount><last-updated>09/10/2015</last-updated><oneWeekChange deprecated="true"></oneWeekChange><valueChange duration="30" currency="USD">20690</valueChange><valuationRange><low currency="USD">1292022</low><high currency="USD">1547586</high></valuationRange><percentile>0</percentile></zestimate><localRealEstate><region name="East Queen Anne" id="271856" type="neighborhood"><zindexValue>629,900</zindexValue><links><overview>http://www.zillow.com/local-info/WA-Seattle/East-Queen-Anne/r_271856/</overview><forSaleByOwner>http://www.zillow.com/east-queen-anne-seattle-wa/fsbo/</forSaleByOwner><forSale>http://www.zillow.com/east-queen-anne-seattle-wa/</forSale></links></region></localRealEstate></result></results></response></SearchResults:searchresults><!-- H:003 T:27ms S:1181 R:Sat Sep 12 23:30:47 PDT 2015 B:4.0.19615-release_20150908-endor.2fa5797~candidate.358c83d -->',
content_type='application/xml',
status=200
)
zillow_data = ZillowWrapper(api_key=None)
deep_search_response = zillow_data.get_deep_search_results(
address, zipcode)
result = GetDeepSearchResults(deep_search_response)
assert result.zillow_id == '48749425'
assert result.home_type == 'SingleFamily'
assert result.home_detail_link == \
'http://www.zillow.com/homedetails/' + \
'2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/'
assert result.graph_data_link == \
'http://www.zillow.com/homedetails/' + \
'2114-Bigelow-Ave-N-Seattle-WA-98109/' + \
'48749425_zpid/#charts-and-data'
assert result.map_this_home_link == \
'http://www.zillow.com/homes/48749425_zpid/'
assert result.tax_year == '2014'
assert result.tax_value == '1060000.0'
assert result.year_built == '1924'
assert result.property_size == '4680'
assert result.home_size == '3470'
assert result.bathrooms == '3.0'
assert result.bedrooms == '4'
assert result.last_sold_date == '11/26/2008'
assert result.last_sold_price_currency == 'USD'
assert result.last_sold_price == '1025000'
lat = float(result.latitude)
assert lat - 0.01 <= 47.637933 <= lat + 0.01
lng = float(result.longitude)
assert lng - 0.01 <= -122.347938 <= lng + 0.01
assert result.zestimate_amount == '1419804'
# assert result.zestimate_currency == 'USD'
assert result.zestimate_last_updated == '09/10/2015'
assert result.zestimate_value_change == '20690'
assert result.zestimate_valuation_range_high == '1547586'
assert result.zestimate_valuationRange_low == '1292022'
assert result.zestimate_percentile == '0'
def __init__(self, zillow_api_key):
if os.path.isfile(ZILLOW_RATE_LIMITER_FILE):
self.rate_limiter = RequestRateLimiter.fromFile(ZILLOW_RATE_LIMITER_FILE)
else:
self.rate_limiter = RequestRateLimiter(max_requests=ZILLOW_QUOTA_MAX_REQUESTS,
time_interval=ZILLOW_QUOTA_TIME_INTERVAL,
state_file=ZILLOW_RATE_LIMITER_FILE)
self.zillow_wrapper = ZillowWrapper(zillow_api_key)
def test_get_updated_property_details_results(self):
"""
"""
zillow_id = '48749425'
zillow_data = ZillowWrapper(self.ZILLOW_API_KEY)
updated_property_details_response = \
zillow_data.get_updated_property_details(zillow_id)
result = GetUpdatedPropertyDetails(updated_property_details_response)
assert result.zillow_id == '48749425'
assert result.home_type == 'SingleFamily'
assert result.home_detail_link == \
'http://www.zillow.com/homedetails/' + \
'2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/'
assert result.photo_gallery == \
'http://www.zillow.com/homedetails/' + \
'2114-Bigelow-Ave-N-Seattle-WA-98109/' + \
'48749425_zpid/#image=lightbox%3Dtrue'
assert result.home_info == \
'http://www.zillow.com/homedetails/' + \
'2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/'
assert result.year_built == '1924'
assert result.property_size == '4680'
assert result.home_size == '3470'
assert result.bathrooms == '3.0'
assert result.bedrooms == '4'
assert result.year_updated == '2003'
assert result.basement == 'Finished'
assert result.roof == 'Composition'
assert result.view == 'Water, City, Mountain'
assert result.heating_sources == 'Gas'
assert result.heating_system == 'Forced air'
assert result.rooms == \
'Laundry room, Walk-in closet, Master bath, Office,' + \
' Dining room, Family room, Breakfast nook'
assert result.neighborhood == 'Queen Anne'
assert result.school_district == 'Seattle'
lat = float(result.latitude)
assert lat - 0.01 <= 47.637933 <= lat + 0.01
lng = float(result.longitude)
assert lng - 0.01 <= -122.347938 <= lng + 0.01
assert result.floor_material is None
assert result.num_floors == '2'
assert result.parking_type == 'Off-street'
def find_zillow_by_zip(address, zip):
zillow_data = ZillowWrapper("X1-ZWz1fe5w83qcjv_70ucn")
try:
deep_search_response = zillow_data.get_deep_search_results(address, zip)
result = GetDeepSearchResults(deep_search_response)
zid = result.zillow_id
#print zid
url = 'http://www.zillow.com/webservice/GetZestimate.htm?zws-id=X1-ZWz1fe5w83qcjv_70ucn&zpid=' + zid
#print url
page = requests.get(url)
tree = html.fromstring(page.content)
prices = tree.xpath('//amount/text()')
zillow = [zid, prices[0]]
return zillow
except ZillowError as err:
print ("No Zillow Found!")
return ['','']
def test_deep_search_results(self):
"""
"""
address = '2114 Bigelow Ave Seattle, WA'
zipcode = '98109'
zillow_data = ZillowWrapper(self.ZILLOW_API_KEY)
deep_search_response = zillow_data.get_deep_search_results(
address, zipcode)
result = GetDeepSearchResults(deep_search_response)
assert result.zillow_id == '48749425'
assert result.home_type == 'SingleFamily'
assert result.home_detail_link == \
'http://www.zillow.com/homedetails/' + \
'2114-Bigelow-Ave-N-Seattle-WA-98109/48749425_zpid/'
assert result.graph_data_link == \
'http://www.zillow.com/homedetails/' + \
'2114-Bigelow-Ave-N-Seattle-WA-98109/' + \
'48749425_zpid/#charts-and-data'
assert result.map_this_home_link == \
'http://www.zillow.com/homes/48749425_zpid/'
assert result.tax_year == '2013'
assert result.tax_value == '995000.0'
assert result.year_built == '1924'
assert result.property_size == '4680'
assert result.home_size == '3470'
assert result.bathrooms == '3.0'
assert result.bedrooms == '4'
assert result.last_sold_date == '11/26/2008'
assert result.last_sold_price_currency == 'USD'
assert result.last_sold_price == '995000'
lat = float(result.latitude)
assert lat - 0.01 <= 47.637933 <= lat + 0.01
lng = float(result.longitude)
assert lng - 0.01 <= -122.347938 <= lng + 0.01
assert result.zestimate_amount == '1433441'
# assert result.zestimate_currency == 'USD'
assert result.zestimate_last_updated == '01/06/2015'
assert result.zestimate_value_change == '7838'
assert result.zestimate_valuation_range_high == '1591120'
assert result.zestimate_valuationRange_low == '1275762'
assert result.zestimate_percentile == '0'
def test_zillow_error_no_property_match(self):
# This test checks the correct error message if no property is found.
# Expected error code: 508
# Address and zip code of an exisiting property, but not listed
address = 'not a valid address'
zipcode = '20001'
zillow_data = ZillowWrapper(self.ZILLOW_API_KEY)
raises(
ZillowError,
zillow_data.get_deep_search_results,
address=address,
zipcode=zipcode)
def test_zillow_error_no_coverage(self):
# This test checks the correct error message
# if no coverage is provided.
# Expected error code: 504
address = 'Calle 21 y O, Vedado, Plaza, Ciudad de la Habana, Cuba'
zipcode = '10400' # Cuban address, I assume Zillow doesn't cover Cuba
zillow_data = ZillowWrapper(self.ZILLOW_API_KEY)
raises(
ZillowError,
zillow_data.get_deep_search_results,
address=address,
zipcode=zipcode)
def zillow_query(address, zipcode, key='YOURAPIKEYHERE'):
"""Function for obtaining data for a given address location
Args:
address (str): street address
zipcode (str): zipcode corresponding with street address
key (str): (default='YOURAPIKEYHERE') zillow api key
Returns:
returns a GetDeepSearchResults object which has following attributes available:
'zillow_id'
'home_type'
'home_detail_link'
'graph_data_link'
'map_this_home_link'
'latitude'
'longitude'
'tax_year'
'tax_value'
'year_built'
'property_size'
'home_size'
'bathrooms'
'bedrooms'
'last_sold_date'
'last_sold_price'
'zestimate_amount'
'zestimate_last_updated'
'zestimate_value_change'
'zestimate_valuation_range_high'
'zestimate_valuationRange_low'
'zestimate_percentile'
"""
zillow_data = ZillowWrapper(key)
deep_search_response = zillow_data.get_deep_search_results(
address, zipcode)
result = GetDeepSearchResults(deep_search_response)
return result
def get_data_from_zillow_api(address, zipcode, API_key):
"""
##################################################################
### This code generates public transportation to a location
### using Google's Directions API.
### _________________________________________________________
### INPUT: address - Redfin Raw Source Data "Address, City, State"
### zipcode - Redfin Raw Source Data "Zipcode"
### API_key - Google API (string)
### OUTPUT: PyZillow class
###################################################################
"""
# Load Zillow API
zillow_data = ZillowWrapper(API_key)
try:
deep_search_response = zillow_data.get_deep_search_results(
address, zipcode)
result = GetDeepSearchResults(deep_search_response)
except:
result = None
return (result)
def find_zillow_by_zip(address, zip):
zillow_data = ZillowWrapper("X1-ZWz1fe5w83qcjv_70ucn")
try:
deep_search_response = zillow_data.get_deep_search_results(
address, zip)
result = GetDeepSearchResults(deep_search_response)
zid = result.zillow_id
#print zid
url = 'http://www.zillow.com/webservice/GetZestimate.htm?zws-id=X1-ZWz1fe5w83qcjv_70ucn&zpid=' + zid
print url
page = requests.get(url)
tree = html.fromstring(page.content)
prices = tree.xpath('//amount/text()')
zillow = [zid, prices[0]]
return zillow
except ZillowError as err:
print("No Zillow Found!")
return ['', '']
except IndexError as err:
print("Index expecption!")
return ['', '']
def get_estimated_price(zillow_id):
"""
Returns the zillow estimated price given a property ID.
"""
zkey = current_app.config['ZKEY']
zillow_data = ZillowWrapper(zkey)
# check if already exists in DB and if so, that data is less than a week old, otherwise call zillow API
db_results = PropertyData.objects(zillow_id=zillow_id)
if db_results.count() > 0 and db_results[0].created_at > datetime.now()-timedelta(days=7):
zestimate_amount = db_results[0].zestimate_amount
else:
try:
detail_result = PropDetailsWithAddressAndZip(zillow_data.get_updated_property_details(zillow_id))
deep_search_result = GetDeepSearchResults(zillow_data.get_deep_search_results(
detail_result.street, detail_result.zip))
zestimate_amount = deep_search_result.zestimate_amount
except ZillowError as err:
return jsonify({'error': err.message, 'estimate': 0})
return jsonify({'estimate': zestimate_amount, 'error': ''})
def __init__(self, loc_x, loc_y):
"""
Constructor method.
loc_x: float
The latitude
loc_y: float
The longitude
"""
# Used to get the address, could be replaced with GoogleAPI class
geolocator = GoogleV3(api_key=googleKey)
locations = geolocator.reverse(str(loc_x) + ", " + str(loc_y))
# Format address for query to Zillow
f = locations[0].address.split(",")
n = re.sub("[^0-9]", "", f[2])
self.address = f[0] + "," + f[1]
self.zipcode = n
self.zillow_data = ZillowWrapper(zillowKey)
def generate_Comp_fields_txt():
f_name = open("UpdatedHomeLatLong.txt", "r")
data_base = open("OnlyCompFields.txt", "+w")
write_out = "street_address | zipcode | zillow_id | Zestimate | Home_type | num_bath | num_bed | home_size | Lat | Long | img url | home description "
data_base.write(write_out)
#f_name=["2893 Springdale Ln, Boulder, CO|80303|13239207|630640|Townhouse|3.5|1|1297|40.009373|-105.255454"]
x = 0
for line in f_name:
line = line.rstrip("\n")
address = line
if x == 100:
break
try:
street_address = re.search("(.*?)(?<=CO)", address)
street_address = street_address.group(0)
zipcode = re.search("(?<=CO\|)(.*?)(?=\|)", address)
zipcode = zipcode.group(0)
#!Get information from first zillow call
zillow_data = ZillowWrapper(zws_id)
deep_search_response = zillow_data.get_deep_search_results(
street_address, zipcode)
result = GetDeepSearchResults(deep_search_response)
#!!The following will be stored as values in database
Home_type = result.home_type
Zestimate = result.zestimate_amount
zillow_id = result.zillow_id
lat = str(result.latitude)
long_res = str(result.longitude)
num_bath = str(result.bathrooms)
num_bed = str(result.bedrooms)
home_size = str(result.home_size)
#######CALL TO UPDATED HOME DETAILS
zillow_data = ZillowWrapper(zws_id)
updated_property_details_response = zillow_data.get_updated_property_details(
zillow_id)
result = GetUpdatedPropertyDetails(
updated_property_details_response)
photo_gall = result.photo_gallery
home_d = result.home_description
write_out = street_address + '|' + zipcode + "|" + zillow_id + "|" + Zestimate + "|" + num_bath + "|" + num_bed + "|" + lat + "|" + long_res + "\n"
print(str(write_out))
data_base.write(write_out)
x += 1
except:
print("Nothing Added")
#f_name.close()
data_base.close()
def Get_Zillow_Data(address, zipcode, Zkey):
from pyzillow.pyzillow import ZillowWrapper, GetDeepSearchResults, GetUpdatedPropertyDetails
zillow_data = ZillowWrapper(Zkey)
deep_search_response = zillow_data.get_deep_search_results(address=address, zipcode=zipcode)
result = GetDeepSearchResults(deep_search_response)
#updated_property_details_response = zillow_data.get_updated_property_details(result.zillow_id)
#updatedresult = GetUpdatedPropertyDetails(updated_property_details_response)
resultdict = {'Zillow ID':result.zillow_id,
'Zestimate':result.zestimate_amount,
'Lot Size':result.property_size,
'Square Feet':result.home_size,
'Bedrooms':result.bedrooms,
'Bathrooms':result.bathrooms,
'Year Built':result.year_built,
'Home Type':result.home_type,
'Listing Link':result.home_detail_link}#,
#'Floors':updatedresult.num_floors,
#'Parking':updatedresult.parking_type,
#'Home Info':updatedresult.home_info,
#'School Info':updatedresult.school_district,
#'Neighborhood Info':updatedresult.neighborhood}
return resultdict
class ZillowPuller(Puller):
def __init__(self, zwid):
super().__init__()
self._headers = ['Zillow Estimate', 'Zillow Last Sold']
self._zillow_data = ZillowWrapper(zwid)
@property
def headers(self):
return self._headers
def parse(self, address):
address, zip_code, _ = re.search(r'(.*)(\d{5}(-\d{4})?)$', address).groups()
result = GetDeepSearchResults(self._zillow_data.get_deep_search_results(address, zip_code))
return {
self.headers[0]: result.zestimate_amount,
self.headers[1]: result.last_sold_date
}
class ZillowAPI(object):
def __init__(self):
self.zillow = ZillowWrapper(settings.ZILLOW_API_KEY)
def lookup(self, address, citystatezip):
try:
return self.ZillowPropertyResults(
self.zillow.get_data(
'http://www.zillow.com/webservice/GetDeepSearchResults.htm',
{
'address': address,
'citystatezip': citystatezip,
'zws-id': self.zillow.api_key,
'rentzestimate': True
}))
except (ZillowError, ZillowFail, ZillowNoResults), e:
return Struct(
dict.fromkeys(
self.ZillowPropertyResults.attribute_mapping.keys()))
def choose(num):
scope = ['https://spreadsheets.google.com/feeds']
credentials = ServiceAccountCredentials.from_json_keyfile_name(
'My Project-f09bdc10eb52.json', scope)
gc = gspread.authorize(credentials)
sh = gc.open_by_url(
"https://docs.google.com/spreadsheets/d/1XOkErm0Oah2zSx2rcJoV0OAGQx8DL-C--iOfZIhXXRs/edit#gid=0"
)
zillow_data = ZillowWrapper("X1-ZWz1fe5w83qcjv_70ucn")
if num == 0:
print "0 -> morris"
morrisentry(sh, zillow_data, 0)
elif num == 1:
print "1 -> essex"
essexentry(sh, zillow_data, 1)
elif num == 2:
print "2 -> bergen"
bergenentry(sh, zillow_data, 2)
bed = result.bedrooms
bath = result.bathrooms
last_sold_date = result.last_sold_date
last_sold_price = result.last_sold_price
house_info.extend(
(zillow_id, address, zipcode, sqft, lot, bed, bath, home_type,
year_built, last_sold_date, last_sold_price))
except:
print(address + ': exception')
return []
return house_info
zillow_data = ZillowWrapper('X1-ZWz18haqkyt2q3_ac8os')
addresses_df = pd.read_csv('data/dublin_addresses.csv')
addresses_array = np.array(
addresses_df[['NUMBER', 'STREET', 'UNIT', 'CITY', 'POSTCODE']])
housing_data_file_name = 'data/dublin_housing_data.csv'
house_address_set = set()
housing_data_list = []
if Path(housing_data_file_name).is_file():
print('file exists')
housing_data_df = pd.read_csv(housing_data_file_name)
house_address_array = np.array(housing_data_df['address'])
house_address_set = set(house_address_array.flat)
def request(zipcode, sort):
"""
This function returns values from Zillow html docs and then parses.
"""
# user agents required for access
req_headers = {
'accept':
'text/html,application/xhtml+xml,application/\
xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'accept-encoding':
'gzip, deflate, br',
'accept-language':
'en-US,en;q=0.8',
'upgrade-insecure-requests':
'1',
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64)AppleWebKit/\
537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
}
cnxt = db.connect(user="******",
password="******",
host="localhost",
database="sys",
charset="utf8mb4")
cursor = cnxt.cursor()
# lost and found - 'X1-ZWz1gtmiat11xn_7ew1d'
# Init api with Zillow access key
API = ZillowWrapper('X1-ZWz186lbvmr8cr_517ck')
content = []
# init url's
with requests.Session() as s:
try:
if sort == "newest":
url = "https://www.zillow.com/homes/for_sale/\
{0}/days_sort".format(zipcode)
elif sort == "cheapest":
url = "https://www.zillow.com/homes/for_sale/\
{0}/pricea_sort/".format(zipcode)
url = url.strip()
print(url)
r = s.get(url, headers=req_headers, timeout=5)
soup = BeautifulSoup(r.content, 'lxml')
# get max page number
## NOTE: Get max page and extract data is broken
max = 0
for i in soup.find_all("a"):
if i.get("aria-label") and i.get("href"):
if (i.get("aria-label")[:4]) == "Page":
if ("" in i.get("href")):
print(i.get("aria-label"))
"""
if i.get('onclick') is not None:
try:
if i.get('onclick'):
if(re.findall('\d+', i.get('onclick'))):
for j in re.findall('\d+', i.get('onclick')):
if max < int(j):
max = int(j)
except IndexError:
pass
"""
hold = url
print(colored("Examining Zillow ..", 'green'))
for i in range(1, max):
if sort == "newest":
url = url + "/" + str(i) + "_p/"
else:
url = url + str(i) + "_p/"
r = s.get(url, headers=req_headers, timeout=5)
soup = BeautifulSoup(r.content, 'lxml')
# Need to find a better algorithm for accessing info from incomplete data
if sort == "cheapest":
address_ref = [
i.text
for i in soup.find_all('span', {'itemprop': 'address'})
]
prices = [
i.text for i in soup.find_all(
'span', {'class': 'zsg-lg-1-4 zsg-right-aligned'})
]
try:
for j in range(len(address_ref)):
if address_ref[j][0].isdigit() is True:
a = address_ref[j].split()
z = a[-1]
a.remove(a[-1])
a = " ".join(a)
print(a)
# Zillow wrapper fails with hashmarks
if "#" not in a:
response = API.get_deep_search_results(
a, z)
result = GetDeepSearchResults(response)
if result.zestimate_amount is not None:
amount = int(result.zestimate_amount)
try:
prices[j] = "$" + \
str(format(amount, ','))
except IndexError:
prices.insert(
j,
"$" + str(format(amount, ',')))
else:
continue
else:
continue
except BaseException as e:
print("API error: %s" % str(e))
print("Defaulting to BeautifulSoup")
prices = [
i.text for i in soup.find_all(
'span',
{'class': 'zsg-lg-1-4 zsg-right-aligned'})
]
break
else:
prices = [
i.text for i in soup.find_all(
'span', {'class': 'zsg-photo-card-price'})
]
info = [
i.text for i in soup.find_all(
'span', {'class': 'zsg-photo-card-info'})
]
address = [
i.text
for i in soup.find_all('span', {'itemprop': 'address'})
]
# creates nested list of Zillow listing
for j, r in enumerate(prices):
content.extend([[prices[j], info[j], address[j]]])
Price = prices[j]
Info = info[j]
Address = address[j]
query = "INSERT INTO PySpider (Price, Info,\
Address) VALUES (%s,%s,%s)"
cursor.execute(query, (Price, Info, Address))
cnxt.commit()
url = hold
print(colored('\n' + "Listings: " + str(len(content)), "green"))
except BaseException as e:
print("An error has occured: %s") % str(e)
def get_house_data_zillow_api(address, zipcode, pull_date):
''' Purpose: Utilizing the zillow api to query data for a single house
Input: Address and zipcode for a single house
Output: None. We will use insert statement within this function'''
# User feedback
print('Accessing Zillow API')
# Instantiate connection to zillow database
web_service_id = 'X1-ZWz1h90xzgg45n_2kpit'
documentation = 'https://pypi.org/project/pyzillow/'
d2 = 'https://anchetawern.github.io/blog/2014/03/20/getting-started-with-zillow-api/'
zillow_data = ZillowWrapper(web_service_id)
deep_search_response = zillow_data.get_deep_search_results(
address, zipcode)
result = GetDeepSearchResults(deep_search_response)
# Create Object for the old & new stdout
old_stdout = sys.stdout
new_stdout = StringIO()
sys.stdout = new_stdout
# Generate stdout - should redirect to StringIO
print('Key, Value')
print('zillow_id,{}'.format(result.zillow_id))
print('home_type,{}'.format(result.home_type))
print('tax_year,{}'.format(result.tax_year))
print('tax_value,{}'.format(result.tax_value))
print('year_built,{}'.format(result.year_built))
print('last_sold_date,{}'.format(result.last_sold_date))
print('last_sold_price,{}'.format(result.last_sold_price))
print('home_size,{}'.format(result.home_size))
print('property_size,{}'.format(result.property_size))
print('num_bedrooms,{}'.format(result.bedrooms))
print('num_bathrooms,{}'.format(result.bathrooms))
print('zillow_low_est,{}'.format(result.zestimate_valuationRange_low))
print('zillow_high_est,{}'.format(result.zestimate_valuation_range_high))
print('zillow_value_change,{}'.format(result.zestimate_value_change))
print('zillow_percentile,{}'.format(result.zestimate_percentile))
# Convert stdout from StringIO to String
new_stdout_str = new_stdout.getvalue()
test_stdout_str = open('zillow_api_data.csv', 'w')
test_stdout_str.write(new_stdout_str)
test_stdout_str.close()
# Return to old stdout
sys.stdout = old_stdout
# Read CSV back in as a pandas dataframe
df = pd.read_csv('zillow_api_data.csv')
df.set_index('Key', inplace=True)
# Need to convert values from strings to correct input value for db
val_zillow_api = []
val_zillow_api.append(address)
val_zillow_api.append(pull_date)
val_zillow_api.append(df.iloc[0, 0]) # zillow_id
val_zillow_api.append(df.iloc[1, 0]) # home type
val_zillow_api.append(df.iloc[2, 0]) # tax year
val_zillow_api.append(df.iloc[3, 0]) # tax value
val_zillow_api.append(df.iloc[4, 0]) # year built
# Try to append formatted datetime object.
try:
val_zillow_api.append(datetime.strptime(df.iloc[5, 0],
'%m/%d/%Y')) # last sold date
# If None date append the arbitrary date of 01/01/1900
except ValueError as err:
print('Zillow api returned the following sold date => {}'.format(
df.iloc[5, 0]))
print('Zillow api value error generated: {}'.format(err))
val_zillow_api.append(datetime.strptime('01/01/1900',
'%m/%d/%Y')) # last sold date
val_zillow_api.append(df.iloc[6, 0]) # last sold price
val_zillow_api.append(df.iloc[7, 0]) # home size
# Test if property value == none. Appaned 0 if None.
if df.iloc[8, 0] == None or df.iloc[8, 0] == 'None':
val_zillow_api.append(0) # property size
else:
val_zillow_api.append(df.iloc[8, 0])
# Continue with other values
val_zillow_api.append(df.iloc[9, 0]) # number bed rooms
val_zillow_api.append(df.iloc[10, 0]) # num bathrooms
val_zillow_api.append(df.iloc[11, 0]) # zillow_low_est
val_zillow_api.append(df.iloc[12, 0]) # zillow high est
val_zillow_api.append(df.iloc[13, 0]) # value change
val_zillow_api.append(df.iloc[14, 0]) # zillow percentile
# Return List Object to be passed to sql insert function
print('Zillow API data successfully obtained\n')
return val_zillow_api
from pyzillow.pyzillow import ZillowWrapper, GetDeepSearchResults
zillow_data = ZillowWrapper('X1-ZWz1iet37nlyiz_4lr3d')
deep_search_response = zillow_data.get_deep_search_results('2114 Bigelow Ave', '98109', True)
result = GetDeepSearchResults(deep_search_response)
print(result.zillow_id)
print(result)
print(result.bathrooms)
want to build experience working with SQL databases, you should refer to the Flask tutorial for instructions on how to
query a SQL database from here instead.
May 2019, Donald Lee-Brown
"""
import sys, os, math, string, time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from flask import render_template
from flaskexample import app
#from flaskexample.a_model import ModelIt
from flask import request
from pyzillow.pyzillow import ZillowWrapper, GetDeepSearchResults
zillow_data = ZillowWrapper("X1-ZWz1b88d9eaq6j_1wtus")
from flaskexample.Model import RenoFeatures
@app.route('/')
def birthmodel_input():
return render_template("index.html")
## Thank you stack overflow for millify function.
## https://stackoverflow.com/questions/3154460/python-human-readable-large-numbers
## user: Janus
def millify(n):
millnames = ['', ' Thousand', ' Million', ' Billion', ' Trillion']
n = float(n)
millidx = max(
class ZillowClient(object):
'''
classdocs
'''
def __init__(self, zillow_api_key):
if os.path.isfile(ZILLOW_RATE_LIMITER_FILE):
self.rate_limiter = RequestRateLimiter.fromFile(ZILLOW_RATE_LIMITER_FILE)
else:
self.rate_limiter = RequestRateLimiter(max_requests=ZILLOW_QUOTA_MAX_REQUESTS,
time_interval=ZILLOW_QUOTA_TIME_INTERVAL,
state_file=ZILLOW_RATE_LIMITER_FILE)
self.zillow_wrapper = ZillowWrapper(zillow_api_key)
@retry(retry_on_exception=isRetryableException,
wait_exponential_multiplier=1000,
wait_exponential_max=10000)
def getDeepSearchResults(self, address, citystatezip, rentzestimate=True):
self.rate_limiter.limit()
deep_search_response = self.zillow_wrapper.get_deep_search_results(
address=address,
zipcode=citystatezip,
rentzestimate=rentzestimate)
return GetDeepSearchResults(deep_search_response)
def updatePropertiesWithZillowData(self, properties):
for prop in properties:
updated_prop = copy.deepcopy(prop)
is_updated = False
result = None
logging.info("Updating: %s, %s, %s",
prop["address"], prop["city"], prop["zip"])
try:
result = self.getDeepSearchResults(
address=prop["address"],
citystatezip=prop["city"] + ", " + prop["zip"])
except ZillowError as e:
logging.error("No Zillow data found for: %s, %s, %s",
prop["address"], prop["city"], prop["zip"])
logging.error("ZillowError: %s", e.message)
# Set invalid zillow-id if no exact match was found for input address
if e.status in [500, 501, 502, 503, 504, 506, 507, 508]:
result = GetDeepSearchResults(
data=ElementTree.fromstring(ZILLOW_NOT_FOUND_RESPONSE))
if result:
for field in ZILLOW_FIELDS:
# Map property fields to Zillow fields.
if field == "zillow_url":
zillow_field = "home_detail_link"
else:
zillow_field = field
if not hasattr(result, zillow_field):
continue
old_value = prop.get(field, "")
new_value = getattr(result, zillow_field)
# Only update fields that have changed.
if (new_value != None and str(new_value) != str(old_value)):
logging.info("%s: %s -> %s", field, old_value, new_value)
updated_prop[field] = new_value
is_updated = True
if is_updated:
logging.info("Updating: %s", prop["address"])
updated_prop["last_update"] = strftime("%Y-%m-%d %H:%M:%S", localtime())
updated_prop["zillow_last_update"] = updated_prop["last_update"]
yield updated_prop, is_updated
__author__ = 'jstannard'
from pyzillow.pyzillow import ZillowWrapper, GetDeepSearchResults, GetUpdatedPropertyDetails
address = '1760 Ardmore St NE'
zipcode = '32907'
# zillow_id = '43504943'
zillow_data = ZillowWrapper('X1-ZWz1a2ngha04jv_8s1xw')
deep_search_response = zillow_data.get_deep_search_results(address, zipcode)
result = GetDeepSearchResults(deep_search_response)
#
# response = zillow_data.ge
# print(result.get_attr('home_info'))
# result = GetUpdatedPropertyDetails(response)
print(result.get_attr('home_size'))
print(result.get_attr('bedrooms'))
print(result.get_attr('home_type'))
print(result.get_attr('zestimate_amount'))
print(result.get_attr('zillow_id'))
url = 'http://www.zillow.com/homes/for_rent/FL/apartment_duplex_type/14_rid/featured_sort/32.630123,-73.487549,22.624152,-92.559814_rect/5_zm/2_p/'
response = requests.get(url)
html = response.content
soup = BeautifulSoup(html, "lxml")
list1 = soup.findAll('span', {'itemprop' : 'streetAddress'})
list2 = soup.findAll('span', {'itemprop' : 'postalCode'})
list10 = []
for item in list1:
text = item.text
list10.append(text)
list11 = []
for item in list2:
text = item.text
list11.append(text)
zillow_data = ZillowWrapper('X1-ZWz1a2ngha04jv_8s1xw')
list6 = []
list7 = []
list9 = []
for i in range (0,len(list10)):
deep_search_response = zillow_data.get_deep_search_results(list10[i], list11[i])
result = GetDeepSearchResults(deep_search_response)
#print(result.get_attr('home_detail_link'))
url2 = result.get_attr('home_detail_link')
response = requests.get(url2)
html = response.content
soup2 = BeautifulSoup(html)
list4 = soup2.findAll('ul', { 'class' : 'zsg-list_square zsg-lg-1-3 zsg-md-1-2 zsg-sm-1-1'})
from pyzillow.pyzillow import ZillowWrapper, GetDeepSearchResults
from settings import ZWSID
zillow_data = ZillowWrapper(ZWSID)
deep_search_response = zillow_data.get_deep_search_results(
'26122 N 121ST AVE,', '85383', True)
result = GetDeepSearchResults(deep_search_response)
print(result.bedrooms)
print(result.bathrooms)
print(result.home_size)
# -*- coding: utf-8 -*- """ Created on Tue May 24 22:06:13 2016 @author: Connor """ import pandas as pd import numpy as np from API_key import * from pyzillow.pyzillow import ZillowWrapper, GetUpdatedPropertyDetails # Well, I think I can just iterate on DC-area zillow IDs maybe? The API requires # specific calls to individual homes, but if I can iterate through them, maybe I # can record the info I want without actually knowing the homes beforehand. # Ha, yes. It seems the Zillow IDs are generated iteratively in some fashion, # so I can at least tease out some basic details on mlutiple home by iterating # over a relatively small range in a a hack-y API-based scrapign loop. zillow_data = ZillowWrapper(ZWSID) updated_property_details_response = zillow_data.get_updated_property_details(4671144) result = GetUpdatedPropertyDetails(updated_property_details_response) #print dir(result) print result.year_built
import numpy as np
ZWSID = 'X1-ZWz1fwjz1gd6a3_8w9nb'
#http://www.zillow.com/webservice/GetDeepSearchResults.htm?zws-id=X1-ZWz1fwjz1gd6a3_8w9nb&address=350+E62nd+Street+Apt+4J&citystatezip=New+York%2C+NY
address1 = '350+E62nd+Street+Apt+4J, New+York, NY'
zip1 = '10065'
#result_deep.zillow_id '31537611'
address2 = '529+Court+Street+Apt+401, Brooklyn, NY'
zip2 = '11231'
#http://www.zillow.com/webservice/GetDeepSearchResults.htm?zws-id=X1-ZWz1fwjz1gd6a3_8w9nb&address=1982+N+Maud+Ave+Apt+i&citystatezip=Chicago%2C+IL
address3 = '1982+N+Maud+Ave+Apt+i, Chicago, IL'
#http://www.zillow.com/webservice/GetDeepSearchResults.htm?zws-id=X1-ZWz1fwjz1gd6a3_8w9nb&address=110+River+Drive+Apt+608&citystatezip=Jersey+City%2C+NJ
address4 = '110+River+Drive+Apt+608, Jersey+City, NJ'
deep_search_response = ZillowWrapper(api_key).get_deep_search_results(
address1, zip1)
result_deep = GetDeepSearchResults(deep_search_response)
#result_updated = GetUpdatedPropertyDetails(deep_search_response)
'''
import requests
from bs4 import BeautifulSoup
url = result_deep.graph_data_link
r = requests.get(url)
'''
link = 'https://www.zillow.com/ajax/homedetail/HomeValueChartData.htm?mt=1&zpid=' + result_deep.zillow_id
raw_file = requests.get(link).content
df = []
for row in raw_file.split('\n'):
df.append(row.split('\t'))
def cities():
street_address = request.args.get('street')
zipcode = request.args.get('zip')
bedrooms = float(request.args.get('bed'))
bathrooms = float(request.args.get('bath'))
home_size = float(request.args.get('size'))
property_size = float(request.args.get('psize'))
year_built = int(request.args.get('year'))
hometype = request.args.get('hometype')
taxyear = int(request.args.get('taxyear'))
taxvalue = float(request.args.get('taxvalue'))
citiesvalues = pd.read_csv('cities-lived.csv') #.to_dict('records')
citiesvalues['user_city'] = 'N'
if street_address != '' and zipcode != '':
search = SearchEngine(simple_zipcode=True)
city = search.by_zipcode(int(zipcode)).city
zillow_data = ZillowWrapper(api_key)
deep_search_response = zillow_data.get_deep_search_results(street_address, zipcode)
result = GetDeepSearchResults(deep_search_response)
errors = []
inputvals = {}
if result.bathrooms is not None and bathrooms == 0:
bathrooms = float(result.bathrooms)
elif result.bathrooms is not None and bathrooms != 0:
bla=0 #do nothing
elif result.bathrooms is None and bathrooms == 0:
errors.append(bathrooms)
inputvals["bathrooms"] = bathrooms
#############################################
if result.bedrooms is not None and bedrooms == 0:
bedrooms = float(result.bedrooms)
elif result.bedrooms is not None and bedrooms != 0:
bla=0 #do nothing
elif result.bedrooms is None and bedrooms == 0:
errors.append(bedrooms)
inputvals["bedrooms"] = bedrooms
#############################################
if result.home_size is not None and home_size == 0:
home_size = float(result.home_size)
elif result.home_size is not None and home_size != 0:
bla=0 #do nothing
elif result.home_size is None and home_size == 0:
errors.append(home_size)
inputvals["home_size"] = home_size
#############################################
if result.property_size is not None and property_size == 0:
property_size = float(result.property_size)
elif result.property_size is not None and property_size != 0:
bla=0 #do nothing
elif result.property_size is None and property_size == 0:
errors.append(property_size)
inputvals["property_size"] = property_size
#############################################
if result.year_built is not None and year_built == 0:
year_built = int(result.year_built)
elif result.year_built is not None and year_built != 0:
bla=0 #do nothing
elif result.year_built is None and year_built == 0:
errors.append(year_built)
inputvals["year_built"] = year_built
#############################################
if result.tax_year is not None and taxyear == 0:
taxyear = int(result.tax_year)
elif result.tax_year is not None and taxyear != 0:
bla=0 #do nothing
elif result.tax_year is None and taxyear == 0:
errors.append(taxyear)
inputvals["taxyear"] = taxyear
#############################################
if result.tax_value is not None and taxvalue == 0:
taxvalue = float(result.tax_value)
elif result.tax_value is not None and taxvalue != 0:
bla=0 #do nothing
elif result.tax_value is None and taxvalue == 0:
errors.append(taxvalue)
inputvals["taxvalue"] = taxvalue
#############################################
if result.home_type is not None and hometype == "Unknown":
hometype = result.home_type
elif result.home_type is not None and hometype != "Unknown":
bla=0 #do nothing
elif result.home_type is None and hometype == "Unknown":
errors.append(hometype)
inputvals["hometype"] = hometype
#############################################
citiesvalues['bedrooms'] = bedrooms
citiesvalues['bathrooms'] = bathrooms
citiesvalues['tax_year'] = taxyear
citiesvalues['tax_value'] = taxvalue
citiesvalues['property_size'] = property_size
citiesvalues['home_size'] = home_size
citiesvalues['location'] = citiesvalues['state'].apply(lambda x: state_mapping[x])
citiesvalues['home_type'] = hometype_mapping[hometype]
citiesvalues['age'] = 2019-year_built
citiesvalues['price'] = model.predict(citiesvalues[['bedrooms','bathrooms','tax_year','tax_value','property_size','home_size','location','home_type','age']])
print(citiesvalues['price'])
citiesvalues['price'] = citiesvalues['price'].apply(lambda x: round(x,2))
citiesvalues = citiesvalues.drop(columns = ['bedrooms','bathrooms','tax_year','tax_value','property_size','home_size','location','home_type','age'])
## for i, r in citiesvalues.iterrows():
## citiesvalues.at[i,'price'] = model.predict(np.array([[bedrooms,bathrooms,taxyear,taxvalue,property_size,home_size,state_mapping[r['state']],hometype_mapping[hometype],2019-year_built]]))[0]
citiesvalues = citiesvalues.to_dict('records')
if result.zestimate_amount is None and result.tax_value is None and result.last_sold_price is not None:
result.zestimate_amount = result.last_sold_price #default for the circle to show
elif result.zestimate_amount is None and result.tax_value is None and result.last_sold_price is None:
result.zestimate_amount = 100000
print({'price':result.zestimate_amount, 'place':city, 'lat':result.latitude, 'lon':result.longitude,'user_city':'Y'})
citiesvalues.append({'price':result.zestimate_amount, 'place':city, 'lat':result.latitude, 'lon':result.longitude,'user_city':'Y'})
citiesvalues.append({"errors":errors,"input":inputvals})
else:
citiesvalues = citiesvalues.to_dict('records')
print(citiesvalues)
return json.dumps(citiesvalues)
Street_Address = DF['Street_Address']
Monthly_Rental = DF['Monthly Rental']
Monthly_Rental = [Monthly_Rental[l] for l in Index]
Z_ID = []
Home_Type = []
Latitude = []
Longitude = []
Home_Size = []
Zest_LD = []
for i in Index:
Address = Street_Address[i]
ZipCode = str(Zip_Code[i])
Zillow_Data = ZillowWrapper('X1-ZWz1fs16znu7m3_2vy2l')
Deep_Search_Response = Zillow_Data.get_deep_search_results(Address, ZipCode)
Result = GetDeepSearchResults(Deep_Search_Response)
Z_ID.append(Result.zillow_id)
Home_Type.append(Result.home_type)
Latitude.append(Result.latitude)
Longitude.append(Result.longitude)
Home_Size.append(Result.home_size)
Zest_LD.append(Result.zestimate_last_updated)
Street_Address = [Street_Address[l] for l in Index]
Zip_Code = [Zip_Code[l] for l in Index]
Columns = {'Serial_Number': Serial_Number, 'Zillow_ID':Z_ID, 'Street_Address':Street_Address, 'Home_Type': Home_Type, 'Latitude':Latitude, 'Longitude': Longitude, 'Home_Size': Home_Size, 'Bathrooms': Bathrooms, 'Bedrooms': Bedrooms, 'Zest_LD' : Zest_LD, 'State': State, 'Days_On_Market': Days_On_Market, 'Zip_Code': Zip_Code, 'Monthly_Rental': Monthly_Rental }
DF_FINAL = pd.DataFrame(Columns)
DF_FINAL.to_csv('Bronx_Zillow_Data_With_Additional_Features.csv')
f_name = open("CompleteAddress.txt", "r")
data_base = open("UpdatedHomeLatLong.txt", "+w")
write_out = "street_address | zipcode | zillow_id | Zestimate | Home_type | num_bath | num_bed | home_size | Lat | Long "
data_base.write(write_out)
for line in f_name:
line = line.rstrip("\n")
address = line
try:
street_address = re.search("(.*?)(?<=CO)", address)
street_address = street_address.group(0)
zipcode = re.search("(?<=CO )(.*?)(?=,)", address)
zipcode = zipcode.group(0)
#!Get information from first zillow call
zillow_data = ZillowWrapper(zws_id)
deep_search_response = zillow_data.get_deep_search_results(
street_address, zipcode)
result = GetDeepSearchResults(deep_search_response)
#!!The following will be stored as values in database
Home_type = result.home_type
Zestimate = result.zestimate_amount
zillow_id = result.zillow_id
lat = str(result.latitude)
long_res = str(result.longitude)
num_bath = str(result.bathrooms)
num_bed = str(result.bedrooms)
home_size = str(result.home_size)
write_out = street_address + '|' + zipcode + "|" + zillow_id + "|" + Zestimate + "|" + Home_type + "|" + num_bath + "|" + num_bed + "|" + home_size + "|" + lat + "|" + long_res + "\n"
def get_house_data_zillow_api(address, zipcode, pull_date, asking_price, url):
try:
# Instantiate connection to zillow database
web_service_id = ''
documentation = ''
d2 = ''
zillow_data = ZillowWrapper(web_service_id)
# Start Search
deep_search_response = zillow_data.get_deep_search_results(
address, zipcode)
result = GetDeepSearchResults(deep_search_response)
# Create Object for the old & new stdout
old_stdout = sys.stdout
new_stdout = StringIO()
sys.stdout = new_stdout
# Generate stdout - should redirect to StringIO
print('Key, Value')
print('zillow_id,{}'.format(result.zillow_id))
print('home_type,{}'.format(result.home_type))
print('tax_year,{}'.format(result.tax_year))
print('tax_value,{}'.format(result.tax_value))
print('year_built,{}'.format(result.year_built))
print('last_sold_date,{}'.format(result.last_sold_date))
print('last_sold_price,{}'.format(result.last_sold_price))
print('home_size,{}'.format(result.home_size))
print('property_size,{}'.format(result.property_size))
print('num_bedrooms,{}'.format(result.bedrooms))
print('num_bathrooms,{}'.format(result.bathrooms))
print('zillow_low_est,{}'.format(result.zestimate_valuationRange_low))
print('zillow_high_est,{}'.format(
result.zestimate_valuation_range_high))
print('zillow_value_change,{}'.format(result.zestimate_value_change))
print('zillow_percentile,{}'.format(result.zestimate_percentile))
# Convert stdout from StringIO to String
new_stdout_str = new_stdout.getvalue()
test_stdout_str = open('zillow_api_data.csv', 'w')
test_stdout_str.write(new_stdout_str)
test_stdout_str.close()
# Return to old stdout
sys.stdout = old_stdout
# Read CSV back in as a pandas dataframe
df = pd.read_csv('zillow_api_data.csv')
df.set_index('Key', inplace=True)
# Need to convert values from strings to correct input value for db
val_zillow_api = []
val_zillow_api.append(address)
val_zillow_api.append(pull_date)
val_zillow_api.append(df.iloc[0, 0]) # zillow_id
val_zillow_api.append(df.iloc[1, 0]) # home type
val_zillow_api.append(df.iloc[2, 0]) # tax year
val_zillow_api.append(df.iloc[3, 0]) # tax value
val_zillow_api.append(df.iloc[4, 0]) # year built
val_zillow_api.append(datetime.strptime('01/01/1900',
'%m/%d/%Y')) # last sold date
val_zillow_api.append(df.iloc[6, 0]) # last sold price
val_zillow_api.append(df.iloc[7, 0]) # home size
# Test if property value == none. Appaned 0 if None.
if df.iloc[8, 0] == None or df.iloc[8, 0] == 'None':
val_zillow_api.append(0) # property size
else:
val_zillow_api.append(df.iloc[8, 0])
# Continue with other values
val_zillow_api.append(df.iloc[9, 0]) # number bed rooms
val_zillow_api.append(df.iloc[10, 0]) # num bathrooms
val_zillow_api.append(df.iloc[11, 0]) # zillow_low_est
val_zillow_api.append(df.iloc[12, 0]) # zillow high est
val_zillow_api.append(df.iloc[13, 0]) # value change
val_zillow_api.append(df.iloc[14, 0]) # zillow percentile
val_zillow_api.append(
asking_price) # value is an input to this function
# Return List Object to be passed to sql insert function
#print('Zillow API data successfully obtained\n')
return val_zillow_api
except IndexError as err:
print('pyzillow generated an error => {}'.format(err))
m1.sql_insert_warning_logs(mydb, 'module_3',
'get_house_data_zillow_api', url,
'Error generated when accessing api',
str(err))
except pyzillow.pyzillowerrors.ZillowError:
print('pyzillow generated an error')
m1.sql_insert_warning_logs(mydb, 'module_3',
'get_house_data_zillow_api', url,
'Error generated when accessing api',
str(err))
except ZillowError as err:
print('Zillow generated an error => {}'.format(err))
m1.sql_insert_warning_logs(mydb, 'module_3',
'get_house_data_zillow_api', url,
'Error generated when accessing', str(err))
def __init__(self, zwid):
super().__init__()
self._headers = ['Zillow Estimate', 'Zillow Last Sold']
self._zillow_data = ZillowWrapper(zwid)
def scrape_zillow_data(input_address):
session = requests.Session()
zillow_data = ZillowWrapper(zillow)
m = re.search(r'[0-9]{5}', input_address)
zipcode = m.group(0)
# print("ZIP code: ", zipcode)
deep_search_response = zillow_data.get_deep_search_results(
input_address, zipcode)
result = GetDeepSearchResults(deep_search_response)
zillow_id = result.zillow_id
# print("Zillow ID: ", zillow_id)
headers = {
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit 537.36 (KHTML, like Gecko) Chrome",
"Accept":
"text/html,application/xhtml+xml,application/xml; q=0.9,image/webp,*/*;q=0.8"
}
url = "https://www.zillow.com/homedetails/" + zillow_id + "_zpid/?fullpage=true"
facts = {}
try:
req = session.get(url, headers=headers)
except:
pass
bsObj = BeautifulSoup(req.text, "lxml")
facts['unit_type'] = bsObj.find(
class_="hdp-fact-ataglance-value").get_text()
try:
unit_facts = bsObj.find("div", {
"class": "hdp-fact-container"
}).find_all("li")
print(unit_facts)
for fact in unit_facts:
fact_name = fact.span.get_text()
try:
fact_value = fact.span.nextSibling.get_text()
except:
fact_value = ""
pass
print(fact_name, fact_value)
except:
print("could not find out unit facts")
try:
unit_dates = bsObj.find(text="Dates").parent.nextSibling.find_all("li")
for date in unit_dates:
m = re.search(r'[12][0-9]{3}', date.get_text())
# print(m.group(0))
facts['dates'] = m.group(0)
except:
print("could not find information on unit dates")
try:
hdp_amen2_facts = bsObj.find(
text="Amenities").parent.nextSibling.find_all("li")
amenities = []
for fact in hdp_amen2_facts:
amenities.append(fact.get_text())
print("amenities:", fact.get_text())
facts['amenities'] = amenities
except:
print("could not find information on unit amenities")
print(facts)
return facts
def get_property_info(address, zipcode): if address and zipcode: zillow_data = ZillowWrapper(ZILLOW_API_KEY) deep_search_response = zillow_data.get_deep_search_results(address, zipcode) result = GetDeepSearchResults(deep_search_response) return result
@author: Michael Wegnerr
@brief: Returns a list of all current rental properties in Boulder
@return: Lsit of rental properties
"""
# importing the requests library
import requests
import xmltodict
from pyzillow.pyzillow import ZillowWrapper, GetDeepSearchResults,GetUpdatedPropertyDetails
from collections import OrderedDict
ZILLOW_API_KEY= ""
address = '890 18th St Boulder, CO 80302'
zipcode = '80302'
zillow_data = ZillowWrapper(ZILLOW_API_KEY)
deep_search_response = zillow_data.get_deep_search_results(address,zipcode)
result = GetDeepSearchResults(deep_search_response)
zillow_id=result.zillow_id
# api-endpoint
URL = "http://www.zillow.com/webservice/GetZestimate.htm"
zws_id=ZILLOW_API_KEY
zpid=zillow_id
rentzestimate=True
# defining a params dict for the parameters to be sent to the API
PARAMS = {'zws-id':zws_id,'zpid':zpid,'rentzestimate':rentzestimate}
#!/usr/bin/env python3
import os
import mysql.connector
from pyzillow.pyzillow import ZillowWrapper, GetDeepSearchResults
address = '8501 Katy Reid Ct.'
zipcode = '23832'
zillow_data = ZillowWrapper(os.environ.get('ZILLOW_API_KEY', ''))
deep_search_response = zillow_data.get_deep_search_results(address, zipcode)
result = GetDeepSearchResults(deep_search_response)
for prop in [
'zillow_id', 'home_type', 'home_detail_link', 'graph_data_link',
'map_this_home_link', 'latitude', 'longitude', 'tax_year', 'tax_value',
'year_built', 'property_size', 'home_size', 'bathrooms', 'bedrooms',
'last_sold_date', 'last_sold_price_currency', 'last_sold_price',
'zestimate_amount', 'zestimate_last_updated', 'zestimate_value_change',
'zestimate_valuation_range_high', 'zestimate_valuationRange_low',
'zestimate_percentile'
]:
value = getattr(result, prop)
print(f'{prop} = {value}')
mydb = mysql.connector.connect(host='127.0.0.1',
user='******',
passwd=os.environ.get('MYSQL_PASSWORD', ''),
database='chris_zillow_example')
try:
mycursor = mydb.cursor()
import os from pyzillow.pyzillow import ZillowWrapper, GetDeepSearchResults, GetUpdatedPropertyDetails ZILLOW_API_KEY = os.environ['ZILLOW_API_KEY'] address = '13680 SW Berea Dr Tigard OR' zipcode = '97223' zillow_data = ZillowWrapper(ZILLOW_API_KEY) deep_search_response = zillow_data.get_deep_search_results(address, zipcode) result = GetDeepSearchResults(deep_search_response) zillow_id = result.zillow_id # zillow id, needed for the GetUpdatedPropertyDetails zillow_data = ZillowWrapper(ZILLOW_API_KEY) updated_property_details_response = zillow_data.get_updated_property_details(zillow_id) result = GetUpdatedPropertyDetails(updated_property_details_response) print result.year_built # number of rooms of the home
def zillow():
"""
returns r_id with a valid search
returns VALID_ZAZ_ERR with invalid search
returns VALID_ZWSID_ERR with missing zwsid
returns VALID_ADDRESS_ERR with missing address
returns VALID_ZIPCODE_ERR with missing zipcode
"""
if request.method in ['POST', 'GET']:
if request.values.get('zwsid'):
zwsid = request.values.get('zwsid')
log.debug(zwsid)
else:
return VALID_ZWSID_ERR
if request.values.get('address'):
address = request.values.get('address')
log.debug(address)
else:
return VALID_ADDRESS_ERR
if request.values.get('zipcode'):
zipcode = request.values.get('zipcode')
log.debug(request)
else:
return VALID_ZIPCODE_ERR
try:
zillow_data = ZillowWrapper(zwsid)
deep_search_response = zillow_data.get_deep_search_results(address, zipcode)
result = GetDeepSearchResults(deep_search_response)
except ZillowError as e:
log.error(e)
return VALID_ZAZ_ERR
client = MongoClient()
db = client.zillow
mongo_result = db.zillow.insert_one(
{
"request_time":datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S'),
"zillow_responce": {
"zillow_id": getattr(result, "zillow_id"),
"home_type": getattr(result, "home_type"),
"home_detail_link": getattr(result, "home_detail_link"),
"graph_data_link": getattr(result, "graph_data_link"),
"map_this_home_link": getattr(result, "map_this_home_link"),
"latitude": getattr(result, "latitude"),
"longitude": getattr(result, "longitude"),
"tax_year": getattr(result, "tax_year"),
"tax_value": getattr(result, "tax_value"),
"year_built": getattr(result, "year_built"),
"property_size": getattr(result, "property_size"),
"home_size": getattr(result, "home_size"),
"bathrooms": getattr(result, "bathrooms"),
"bedrooms": getattr(result, "bedrooms"),
"last_sold_date": getattr(result, "last_sold_date"),
"last_sold_price_currency": getattr(result, "last_sold_price_currency"),
"last_sold_price": getattr(result, "last_sold_price"),
"zestimate_amount": getattr(result, "zestimate_amount"),
"zestimate_last_updated": getattr(result, "zestimate_last_updated"),
"zestimate_value_change": getattr(result, "zestimate_value_change"),
"zestimate_valuation_range_high": getattr(result, "zestimate_valuation_range_high"),
"zestimate_valuationRange_low": getattr(result, "zestimate_valuationRange_low"),
"zestimate_percentile": getattr(result, "zestimate_percentile")
},
"request_information": {
'zwsid':zwsid,
'address': address,
'zipcode': zipcode
},
}
)
return str(mongo_result.inserted_id)
return VALID_ZAZ_ERR
today_data = Data[str(today)]
#myData is a list of Dicts
Final_data = pd.DataFrame()
count = 0
api_count = 0
api_keys = [
'X1-ZWz17l7hzioj63_3pg0i', 'X1-ZWz17mttceiwwb_2599m',
'X1-ZWz1hfb2wz6ia3_3qukz'
]
##next_day = datetime.datetime.combine(next_day,datetime.datetime.min.time()) + datetime.timedelta(hours=2)
new = []
for each_address in today_data:
street_address = each_address['NUMBER'] + ' ' + each_address['STREET']
zipcode = each_address['POSTCODE']
zillow_data = ZillowWrapper(api_keys[api_count])
try:
count = count + 1 #Being conservative here , i want to count even if the address failed
deep_search_response = zillow_data.get_deep_search_results(
street_address, zipcode)
result = GetDeepSearchResults(deep_search_response)
except:
continue
mydict = {}
mydict['Address'] = street_address
mydict['ZIP'] = zipcode
mydict["zillow_id"] = result.zillow_id
mydict["home_type"] = result.home_type
mydict["year_built"] = result.year_built
mydict["property_size"] = result.property_size
mydict["home_size"] = result.home_size
def __init__(self):
self.zillow = ZillowWrapper(settings.ZILLOW_API_KEY)
from pyzillow.pyzillow import ZillowWrapper, GetDeepSearchResults
address = 'address goes here'
zipcode = 'zip goes here'
zillow_data = ZillowWrapper('api token')
deep_search_response = zillow_data.get_deep_search_results(address, zipcode)
result = GetDeepSearchResults(deep_search_response)
print(result.bathrooms)
def main():
## Zillow API call setup ##
API_KEY = 'X1-ZWz1fmcredpjpn_76gmj'
zillow_data = ZillowWrapper(API_KEY)
## variable setup for iteration through address list (from file) ##
i = 0
while i < len(address_list):
## check used to determine if ds_result raised an exception - no data
check = True
## upd used to determine if upd_result raised an exception - no data
upd = True
address = address_list[i]
if (check_duplicates(address)):
i += 1
exit
else:
## First API call using address/zip ##
try:
ds_response = zillow_data.get_deep_search_results(
address, '27401')
ds_result = GetDeepSearchResults(ds_response)
except (ZillowError, UnicodeEncodeError):
print('No basic property details for:' + address)
check = False
finally:
exit
## If first API call successful, additional details sought from 2nd API call
if check:
try:
upd_response = zillow_data.get_updated_property_details(
ds_result.zillow_id)
upd_result = GetUpdatedPropertyDetails(upd_response)
except (ZillowError, UnicodeEncodeError):
print('No updated property details for:' + address)
upd = False
finally:
exit
## Compares sold date vs. current date to determine "sold" binary status
## Older date is "less than" for Python comparison ##
## We treat anything sold more than 1 year ago as "unsold" ##
if check:
if upd:
t = 1
if str(ds_result.last_sold_date) != 'None':
sale_date = dt.strptime(ds_result.last_sold_date,
"%m/%d/%Y")
compare_date = dt.strptime('01/01/2016', "%m/%d/%Y")
if sale_date < compare_date:
s = 0
else:
s = 1
else:
s = 0
insert_home(
address, 27401, 'Greensboro', 'NC',
ds_result.home_type, ds_result.bedrooms,
ds_result.bathrooms, ds_result.home_size,
ds_result.property_size, ds_result.year_built,
upd_result.year_updated, upd_result.num_floors,
upd_result.parking_type, upd_result.heating_sources,
upd_result.heating_system, upd_result.floor_material,
upd_result.num_rooms, str(upd_result.neighborhood),
str(upd_result.school_district), s,
ds_result.last_sold_date, ds_result.last_sold_price,
upd_result.appliances, upd_result.roof,
upd_result.rooms, t)
else:
s = 0
t = 0
default = None
insert_home(address, 27401, 'Greensboro', 'NC',
ds_result.home_type, ds_result.bedrooms,
ds_result.bathrooms, ds_result.home_size,
ds_result.property_size, ds_result.year_built,
default, default, default, default, default,
default, default, default, default, s, default,
default, default, default, default, t)
i += 1