def handle_message(msg):
task = json.loads(msg)
if (not isinstance(task, dict) or not 'zpid' in task
or task['zpid'] is None):
return
zpid = task['zpid']
#Scrape the zillow for details( decouple system )
property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid)
# update doc in db
db = mongodb_client.getDB()
# if exists, then replace, if not exists, then insert
db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid},
property_detail,
upsert=True)
if FETCH_SIMILAR_PROPERTIES:
# get its similar properties' zpid
similar_zpids = zillow_web_scraper_client.get_similar_homes_for_sale_by_zpid(
zpid)
# generate tasks for similar zpids
for zpid in similar_zpids:
# avoid loop tasks
db_homes = db[PROPERTY_TABLE_NAME].find_one({{'zpid': zpid}})
if (db_homes is not None and
time.time() - db_homes['last_update'] < SECONDS_ONE_WEEK):
continue
client.sendDataFetcherTask({'zpid': zpid})
def searchArea(text):
zpids = []
if text.isdigit():
zpids = searchAreaByZip(text)
else:
city = text.split(',')[0].strip()
state = text.split(', ')[1].strip()
zpids = searchAreaByCityState(city, state)
print zpids
res = []
update_list = []
db = mongodb_client.getDB()
for zpid in zpids:
record = db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})
if record != None:
res.append(record)
else:
property_detail = getDetailsByZpid(zpid, False)
res.append(property_detail)
update_list.append(property_detail)
storeUpdates(update_list)
# Trick: use bson.dumps then re-construct json because of ObjectId.
return json.loads(dumps(res))
def sendEmail(email, test):
_user = "******"
_pwd = "rhgkgyzoamklbdae"
_to = email
zpid = test['zpid']
street_address = test['street_address']
city = test['city']
state = test['state']
zipcode = test['zipcode']
list_price = test['list_price']
db = mongodb_client.getDB(USERLIST_DATABASE)
userlist = db[USERLIST_TABLE_NAME].find_one({'email': email, 'zpid': zpid})
print userlist
old_list_price = userlist['list_price']
massage = "the house which zpid is %s and list_price was %s, now the list_price is %s" % (
zpid, list_price, old_list_price)
msg = MIMEText(massage)
msg["Subject"] = "Price Change Notation for your Watchlist in Smart Zillow"
msg["From"] = _user
msg["To"] = _to
try:
s = smtplib.SMTP_SSL("smtp.qq.com", 465)
s.login(_user, _pwd)
s.sendmail(_user, _to, msg.as_string())
s.quit()
print "Success!"
except smtplib.SMTPException, e:
print "Falied,%s" % e
def findProperyByZipcode(zipcode):
db = mongodb_client.getDB()
properties = list(db[PROPERTY_TABLE_NAME].find({
'zipcode': zipcode,
'is_for_sale': True
}))
return [x['zpid'] for x in properties]
def handle_message(msg):
task = json.loads(msg)
if (not isinstance(task, dict) or not 'zpid' in task
or task['zpid'] is None):
return
zpid = task['zpid']
# Scrape the zillow for details
property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid)
# Add timestamp
property_detail['last_update'] = time.time()
# update doc in db
db = mongodb_client.getDB()
db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid},
property_detail,
upsert=True)
print property_detail
if FETCH_SIMILAR_PROPERTIES:
# get its similar propertie's zpid
similar_zpids = zillow_web_scraper_client.get_similar_homes_for_sale_by_id(
zpid)
# generate taslks for similar zpids
for zpid in similar_zpids:
old = db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})
# Don't send task if the record is recent
if (old is not None and 'last_update' in old and
time.time() - old['last_update'] < SECONDS_IN_ONE_WEEK):
continue
cloudAMQP_client.sendDataFetcherTask({'zpid': zpid})
def getDetailsByZpid(zpid):
print "getDetailsByZillowId() gets called with zpid=[%s]" % zpid
db = mongodb_client.getDB()
prop = json.loads(dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})))
if prop == None:
prop = zillow_web_scraper_client.get_property_by_zpid(zpid)
return prop
def getPermissionList():
userDB = mongodb_client.getDB(USERS_PERMISSIN_DATABASE)
sig = True
users = []
for user in userDB[USERS_EMAILSENDER_PERMISSION].find(
{'emailSend_permission': sig}):
users.append(user['email'])
return users
def storeUpdates(properties):
print "updating properties in db after searching..."
db = mongodb_client.getDB()
for property_detail in properties:
zpid = property_detail['zpid']
property_detail['last_update'] = time.time()
db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid},
property_detail,
upsert=True)
def findPropertyByZipcode(zipcode):
print "findPropertyByZipcode() gets called with zipcode=[%s]" % str(
zipcode)
db = mongodb_client.getDB()
properties = list(db[PROPERTY_TABLE_NAME].find({
'zipcode': zipcode,
'is_for_sale': True
}))
# Trick: ObjectId is not JSON serializable
return json.loads(dumps(properties))
def findPropertyByCityState(city, state):
print "findPropertyByCityState() gets called with city=[%s] abd state=[%s]" % (
city, state)
db = mongodb_client.getDB()
# use regexp to do case-insensitive search
properties = list(db[PROPERTY_TABLE_NAME]\
.find({ 'city': re.compile(city, re.IGNORECASE),\
'state': re.compile(state, re.IGNORECASE),\
'is_for_sale': True}))
return json.loads(dumps(properties))
def searchArea(self, query):
res = []
if query.isdigit():
print "zipcode"
db = mongodb_client.getDB()
res = db[PROPERTY_TABLE_NAME].find({'zipcode': query})
res = json.loads(dumps(res))
else:
city = query.split(',')[0].strip()
state = query.split(',')[1].strip()
#TODO
return res
def getWatchList(email):
db = mongodb_client.getDB()
watchlist_docs = db[WATCHLIST_TABLE_NAME].find({
'email': email
}).sort("created_at", pymongo.DESCENDING)
properties = []
for doc in watchlist_docs:
zpid = doc['zpid']
property_detail = db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})
property_detail['created_price'] = doc['created_price']
properties.append(property_detail)
return json.loads(dumps(properties))
def getDetailsByZpid(zpid, get_prediction=False):
db = mongodb_client.getDB()
prop = json.loads(dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})))
if prop == None:
prop = zillow_web_scraper_client.get_property_by_zpid(zpid)
# Get prediction
if get_prediction:
predicted_value = ml_prediction_client.predict(
prop['zipcode'], prop['property_type'], prop['bedroom'],
prop['bathroom'], prop['size'], prop['longitude'],
prop['latitude'])
prop['predicted_value'] = int(predicted_value)
return prop
def updateWatchList(email):
db = mongodb_client.getDB()
watchlist_docs = db[WATCHLIST_TABLE_NAME].find({'email': email})
for doc in watchlist_docs:
zpid = doc['zpid']
print "\n > updating watchlist for " + zpid
# update
property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid)
property_detail['last_update'] = time.time()
db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid},
property_detail,
upsert=True) # use replace_one
print property_detail
time.sleep(WAITING_TIME)
return "success"
def getDetailsByZpid(zpid, get_prediction=False):
db = mongodb_client.getDB()
prop = json.loads(dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})))
if prop == None or prop['zestimate'] == None:
prop = zillow_web_scraper_client.get_property_by_zpid(zpid)
##Get prediction
if get_prediction:
prop['lotsize'] = getLot(prop)
predicted_value = ml_prediction_client.predict(
prop['property_type'], prop['bedroom'], prop['bathroom'],
prop['geohash'], prop['school_ratingE'], prop['school_ratingH'],
prop['school_ratingM'], prop['size'], prop['zestimate'],
prop['lotsize'])
prop['predicted_value'] = int(predicted_value)
return prop
def getDetailsByZpid(zpid, get_prediction=False):
print "getDetailsByZpid() gets called with zpid=[%s]" % str(zpid)
db = mongodb_client.getDB()
property_detail = json.loads(
dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid})))
if property_detail is None:
property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid)
# prediciton!
if get_prediction:
print "getting prediction"
predicted_value = ml_prediction_client.predict(
property_detail['zipcode'], property_detail['property_type'],
property_detail['bedroom'], property_detail['bathroom'],
property_detail['size'])
property_detail['predicted_value'] = int(predicted_value)
return property_detail
def getUserList(email):
db = mongodb_client.getDB('real-estate-smart-view')
userlists = json.loads(
dumps(db[USERLIST_TABLE_NAME].find({'email': email})))
return userlists
import mongodb_client
db=mongodb_client.getDB()
db.test.insert({"test":"testvalue"})
print list(db.test.find({'test':"testvalue"}))
def deleteItem(email, zpid):
db = mongodb_client.getDB('real-estate-smart-view')
result = db[USERLIST_TABLE_NAME].delete_one({'email': email, 'zpid': zpid})
return result.deleted_count
