def handle_message(msg):
task = json.loads(msg)
if (not isinstance(task, dict) or not 'zpid' in task
or task['zpid'] is None):
return
zpid = task['zpid']
# Scrape the zillow for details
property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid)
# Add timestamp
property_detail['last_update'] = time.time()
# update doc in db
db = mongodb_client.getDB()
db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid},
property_detail,
upsert=True)
print property_detail
if FETCH_SIMILAR_PROPERTIES:
# get its similar propertie's zpid
similar_zpids = zillow_web_scraper_client.get_similar_homes_for_sale_by_id(
zpid)
# generate taslks for similar zpids
for zpid in similar_zpids:
old = db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})
# Don't send task if the record is recent
if (old is not None and 'last_update' in old and
time.time() - old['last_update'] < SECONDS_IN_ONE_WEEK):
continue
cloudAMQP_client.sendDataFetcherTask({'zpid': zpid})
def handle_message(msg):
task = json.loads(msg)
if (not isinstance(task, dict) or not 'zpid' in task
or task['zpid'] is None):
return
zpid = task['zpid']
#Scrape the zillow for details( decouple system )
property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid)
# update doc in db
db = mongodb_client.getDB()
# if exists, then replace, if not exists, then insert
db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid},
property_detail,
upsert=True)
if FETCH_SIMILAR_PROPERTIES:
# get its similar properties' zpid
similar_zpids = zillow_web_scraper_client.get_similar_homes_for_sale_by_zpid(
zpid)
# generate tasks for similar zpids
for zpid in similar_zpids:
# avoid loop tasks
db_homes = db[PROPERTY_TABLE_NAME].find_one({{'zpid': zpid}})
if (db_homes is not None and
time.time() - db_homes['last_update'] < SECONDS_ONE_WEEK):
continue
client.sendDataFetcherTask({'zpid': zpid})
def getDetailsByZpid(zpid):
print "getDetailsByZillowId() gets called with zpid=[%s]" % zpid
db = mongodb_client.getDB()
prop = json.loads(dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})))
if prop == None:
prop = zillow_web_scraper_client.get_property_by_zpid(zpid)
return prop
def getDetailsByZpid(zpid, get_prediction=False):
db = mongodb_client.getDB()
prop = json.loads(dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})))
if prop == None:
prop = zillow_web_scraper_client.get_property_by_zpid(zpid)
# Get prediction
if get_prediction:
predicted_value = ml_prediction_client.predict(
prop['zipcode'], prop['property_type'], prop['bedroom'],
prop['bathroom'], prop['size'], prop['longitude'],
prop['latitude'])
prop['predicted_value'] = int(predicted_value)
return prop
def searchAreaByZipcode(zipcode):
print "searchAreaByZipcode() gets called with zipcode=[%s]" % str(zipcode)
properties = findPropertyByZipcode(zipcode) #rename
if len(properties) == 0:
# cannot find in db, use scraper to fetch
zpids = zillow_web_scraper_client.get_zpid_by_zipcode(
zipcode) # rename
for zpid in zpids:
property_detail = zillow_web_scraper_client.get_property_by_zpid(
zpid)
properties.append(property_detail)
# update db
storeUpdates(properties)
return properties
def searchAreaByCityState(city, state):
print "searchAreaByCityState() gets called with city=[%s] abd state=[%s]" % (
city, state)
properties = findPropertyByCityState(city, state)
if len(properties) == 0:
# cannot find in db, use scraper to fetch
zpids = zillow_web_scraper_client.get_zpid_by_city_state(
city, state) # rename
for zpid in zpids:
property_detail = zillow_web_scraper_client.get_property_by_zpid(
zpid)
properties.append(property_detail)
# update db
storeUpdates(properties)
return properties
def updateWatchList(email):
db = mongodb_client.getDB()
watchlist_docs = db[WATCHLIST_TABLE_NAME].find({'email': email})
for doc in watchlist_docs:
zpid = doc['zpid']
print "\n > updating watchlist for " + zpid
# update
property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid)
property_detail['last_update'] = time.time()
db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid},
property_detail,
upsert=True) # use replace_one
print property_detail
time.sleep(WAITING_TIME)
return "success"
def handle_message(msg):
task = json.loads(msg)
#task = msg
if (not isinstance(task, dict) or not 'zpid' in task
or task['zpid'] is None):
return
zpid = task['zpid']
users = getPermissionList()
# Scrape the zillow for details
property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid)
for user in users:
emailSender.sendEmail(user, property_detail)
def getDetailsByZpid(zpid, get_prediction=False):
db = mongodb_client.getDB()
prop = json.loads(dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})))
if prop == None or prop['zestimate'] == None:
prop = zillow_web_scraper_client.get_property_by_zpid(zpid)
##Get prediction
if get_prediction:
prop['lotsize'] = getLot(prop)
predicted_value = ml_prediction_client.predict(
prop['property_type'], prop['bedroom'], prop['bathroom'],
prop['geohash'], prop['school_ratingE'], prop['school_ratingH'],
prop['school_ratingM'], prop['size'], prop['zestimate'],
prop['lotsize'])
prop['predicted_value'] = int(predicted_value)
return prop
def getDetailsByZpid(zpid, get_prediction=False):
print "getDetailsByZpid() gets called with zpid=[%s]" % str(zpid)
db = mongodb_client.getDB()
property_detail = json.loads(
dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})))
if property_detail is None:
property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid)
# prediciton!
if get_prediction:
print "getting prediction"
predicted_value = ml_prediction_client.predict(
property_detail['zipcode'], property_detail['property_type'],
property_detail['bedroom'], property_detail['bathroom'],
property_detail['size'])
property_detail['predicted_value'] = int(predicted_value)
return property_detail
import zillow_web_scraper_client as client
print client.search_zillow_by_zip("94015")
print client.search_zillow_by_city_state("San Francisco", "CA")
print client.get_property_by_zpid(83154148)
print client.get_properties_by_zip(94080)
print client.get_properties_by_city_state('San Bruno', 'CA')
print client.get_similar_homes_for_sale_by_id(2096630311)
import zillow_web_scraper_client as client print client.get_property_by_zpid(2114560913) #2114560913
import zillow_web_scraper_client as client client.get_property_by_zpid(48956480)
import zillow_web_scraper_client as client
print client.get_property_by_zpid("94015")
print client.get_property_by_zpid("94037")