def handle_message(msg): task = json.loads(msg) if (not isinstance(task, dict) or not 'zpid' in task or task['zpid'] is None): return zpid = task['zpid'] #Scrape the zillow for details( decouple system ) property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid) # update doc in db db = mongodb_client.getDB() # if exists, then replace, if not exists, then insert db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid}, property_detail, upsert=True) if FETCH_SIMILAR_PROPERTIES: # get its similar properties' zpid similar_zpids = zillow_web_scraper_client.get_similar_homes_for_sale_by_zpid( zpid) # generate tasks for similar zpids for zpid in similar_zpids: # avoid loop tasks db_homes = db[PROPERTY_TABLE_NAME].find_one({{'zpid': zpid}}) if (db_homes is not None and time.time() - db_homes['last_update'] < SECONDS_ONE_WEEK): continue client.sendDataFetcherTask({'zpid': zpid})
def searchArea(text): zpids = [] if text.isdigit(): zpids = searchAreaByZip(text) else: city = text.split(',')[0].strip() state = text.split(', ')[1].strip() zpids = searchAreaByCityState(city, state) print zpids res = [] update_list = [] db = mongodb_client.getDB() for zpid in zpids: record = db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid}) if record != None: res.append(record) else: property_detail = getDetailsByZpid(zpid, False) res.append(property_detail) update_list.append(property_detail) storeUpdates(update_list) # Trick: use bson.dumps then re-construct json because of ObjectId. return json.loads(dumps(res))
def sendEmail(email, test): _user = "******" _pwd = "rhgkgyzoamklbdae" _to = email zpid = test['zpid'] street_address = test['street_address'] city = test['city'] state = test['state'] zipcode = test['zipcode'] list_price = test['list_price'] db = mongodb_client.getDB(USERLIST_DATABASE) userlist = db[USERLIST_TABLE_NAME].find_one({'email': email, 'zpid': zpid}) print userlist old_list_price = userlist['list_price'] massage = "the house which zpid is %s and list_price was %s, now the list_price is %s" % ( zpid, list_price, old_list_price) msg = MIMEText(massage) msg["Subject"] = "Price Change Notation for your Watchlist in Smart Zillow" msg["From"] = _user msg["To"] = _to try: s = smtplib.SMTP_SSL("smtp.qq.com", 465) s.login(_user, _pwd) s.sendmail(_user, _to, msg.as_string()) s.quit() print "Success!" except smtplib.SMTPException, e: print "Falied,%s" % e
def findProperyByZipcode(zipcode): db = mongodb_client.getDB() properties = list(db[PROPERTY_TABLE_NAME].find({ 'zipcode': zipcode, 'is_for_sale': True })) return [x['zpid'] for x in properties]
def handle_message(msg): task = json.loads(msg) if (not isinstance(task, dict) or not 'zpid' in task or task['zpid'] is None): return zpid = task['zpid'] # Scrape the zillow for details property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid) # Add timestamp property_detail['last_update'] = time.time() # update doc in db db = mongodb_client.getDB() db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid}, property_detail, upsert=True) print property_detail if FETCH_SIMILAR_PROPERTIES: # get its similar propertie's zpid similar_zpids = zillow_web_scraper_client.get_similar_homes_for_sale_by_id( zpid) # generate taslks for similar zpids for zpid in similar_zpids: old = db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid}) # Don't send task if the record is recent if (old is not None and 'last_update' in old and time.time() - old['last_update'] < SECONDS_IN_ONE_WEEK): continue cloudAMQP_client.sendDataFetcherTask({'zpid': zpid})
def getDetailsByZpid(zpid): print "getDetailsByZillowId() gets called with zpid=[%s]" % zpid db = mongodb_client.getDB() prop = json.loads(dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid}))) if prop == None: prop = zillow_web_scraper_client.get_property_by_zpid(zpid) return prop
def getPermissionList(): userDB = mongodb_client.getDB(USERS_PERMISSIN_DATABASE) sig = True users = [] for user in userDB[USERS_EMAILSENDER_PERMISSION].find( {'emailSend_permission': sig}): users.append(user['email']) return users
def storeUpdates(properties): print "updating properties in db after searching..." db = mongodb_client.getDB() for property_detail in properties: zpid = property_detail['zpid'] property_detail['last_update'] = time.time() db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid}, property_detail, upsert=True)
def findPropertyByZipcode(zipcode): print "findPropertyByZipcode() gets called with zipcode=[%s]" % str( zipcode) db = mongodb_client.getDB() properties = list(db[PROPERTY_TABLE_NAME].find({ 'zipcode': zipcode, 'is_for_sale': True })) # Trick: ObjectId is not JSON serializable return json.loads(dumps(properties))
def findPropertyByCityState(city, state): print "findPropertyByCityState() gets called with city=[%s] abd state=[%s]" % ( city, state) db = mongodb_client.getDB() # use regexp to do case-insensitive search properties = list(db[PROPERTY_TABLE_NAME]\ .find({ 'city': re.compile(city, re.IGNORECASE),\ 'state': re.compile(state, re.IGNORECASE),\ 'is_for_sale': True})) return json.loads(dumps(properties))
def searchArea(self, query): res = [] if query.isdigit(): print "zipcode" db = mongodb_client.getDB() res = db[PROPERTY_TABLE_NAME].find({'zipcode': query}) res = json.loads(dumps(res)) else: city = query.split(',')[0].strip() state = query.split(',')[1].strip() #TODO return res
def getWatchList(email): db = mongodb_client.getDB() watchlist_docs = db[WATCHLIST_TABLE_NAME].find({ 'email': email }).sort("created_at", pymongo.DESCENDING) properties = [] for doc in watchlist_docs: zpid = doc['zpid'] property_detail = db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid}) property_detail['created_price'] = doc['created_price'] properties.append(property_detail) return json.loads(dumps(properties))
def getDetailsByZpid(zpid, get_prediction=False): db = mongodb_client.getDB() prop = json.loads(dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid}))) if prop == None: prop = zillow_web_scraper_client.get_property_by_zpid(zpid) # Get prediction if get_prediction: predicted_value = ml_prediction_client.predict( prop['zipcode'], prop['property_type'], prop['bedroom'], prop['bathroom'], prop['size'], prop['longitude'], prop['latitude']) prop['predicted_value'] = int(predicted_value) return prop
def updateWatchList(email): db = mongodb_client.getDB() watchlist_docs = db[WATCHLIST_TABLE_NAME].find({'email': email}) for doc in watchlist_docs: zpid = doc['zpid'] print "\n > updating watchlist for " + zpid # update property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid) property_detail['last_update'] = time.time() db[PROPERTY_TABLE_NAME].replace_one({'zpid': zpid}, property_detail, upsert=True) # use replace_one print property_detail time.sleep(WAITING_TIME) return "success"
def getDetailsByZpid(zpid, get_prediction=False): db = mongodb_client.getDB() prop = json.loads(dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid}))) if prop == None or prop['zestimate'] == None: prop = zillow_web_scraper_client.get_property_by_zpid(zpid) ##Get prediction if get_prediction: prop['lotsize'] = getLot(prop) predicted_value = ml_prediction_client.predict( prop['property_type'], prop['bedroom'], prop['bathroom'], prop['geohash'], prop['school_ratingE'], prop['school_ratingH'], prop['school_ratingM'], prop['size'], prop['zestimate'], prop['lotsize']) prop['predicted_value'] = int(predicted_value) return prop
def getDetailsByZpid(zpid, get_prediction=False): print "getDetailsByZpid() gets called with zpid=[%s]" % str(zpid) db = mongodb_client.getDB() property_detail = json.loads( dumps(db[PROPERTY_TABLE_NAME].find_one({'zpid': zpid}))) if property_detail is None: property_detail = zillow_web_scraper_client.get_property_by_zpid(zpid) # prediciton! if get_prediction: print "getting prediction" predicted_value = ml_prediction_client.predict( property_detail['zipcode'], property_detail['property_type'], property_detail['bedroom'], property_detail['bathroom'], property_detail['size']) property_detail['predicted_value'] = int(predicted_value) return property_detail
def getUserList(email): db = mongodb_client.getDB('real-estate-smart-view') userlists = json.loads( dumps(db[USERLIST_TABLE_NAME].find({'email': email}))) return userlists
import mongodb_client db=mongodb_client.getDB() db.test.insert({"test":"testvalue"}) print list(db.test.find({'test':"testvalue"}))
def deleteItem(email, zpid): db = mongodb_client.getDB('real-estate-smart-view') result = db[USERLIST_TABLE_NAME].delete_one({'email': email, 'zpid': zpid}) return result.deleted_count