class Simulator:
def __init__(self):
self.dao = PropertiesDao()
self.loadModels()
def loadModels(self):
self.path = 'data/'
self.dateLoaded = datetime.now()
self.models = []
modelNames = [
'XGBRegressor', 'GradientBoostingRegressor', 'OLS Regression',
'DMatrix Regression', 'XGB Linear Regression'
]
for name in modelNames:
model = self.dao.getLastModel(name)
if model is not None:
self.models.append(model)
self.modelsApp = []
for model in self.models:
name = model['model']
print('Loading Model ', name)
model_reg = pickle.load(open(self.path + model['file'], 'rb'))
model_json = {'key': name, 'model': model_reg}
self.modelsApp.append(model_json)
print('----- Models Loaded Successfully -----')
def simulate(self, record):
try:
record['size_sqft_lg'] = np.log10(record['size_sqft'])
df = record[self.dao.getDataSetModelNames()]
for model in self.modelsApp:
key = model['key']
reg = model['model']
if key == 'DMatrix Regression':
dtest = xg.DMatrix(df, label='')
pred = reg.predict(dtest)
else:
pred = reg.predict(df)
record[key] = pred.tolist()
return record
except Exception as ex:
print('Error simulating models ', ex)
import nltk # Load NLTK
import time
from text_mining_service.sklearn_service import SkLearnService
from text_mining_service.property_service import PropertyService
from db.propertiesdao import PropertiesDao
# nltk.download()
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
dao = PropertiesDao()
propertyService = PropertyService()
sklearnService = SkLearnService()
rows = dao.getAllPropertiesWithQuery(
{"$or": [{
"bedrooms": None
}, {
"bedrooms": 0
}]})
print("Loaded Records: ", len(rows))
size = len(rows)
count = 0
for property in rows:
property = propertyService.tryGetBedroomFromDescription(property)
if property['update'] == True:
print(property['_id'], ' - ', property['bedrooms'], 'bdr ')
import pandas as pd
import numpy as np
from db.propertiesdao import PropertiesDao
from pred_models.simulator import Simulator
import json
#simulator = Simulator()
dao = PropertiesDao()
# intialise data of lists.
data = {'price':[2400.0],
'bedrooms':[3.0],
'bath':[3.0],
'size_sqft':[1479.0],
'professionally_managed':[0.0],
'no_pet_allowed':[1.0],
'suit_laundry':[1.0],
'park_stall':[1.0],
'available_now':[0.0],
'furnished':[1.0],
'amenities':[0.0],
'brand_new':[0.0],
'loc_vancouver':[1.0],
'loc_burnaby':[0.0],
'loc_richmond':[0.0],
'loc_surrey':[1.0],
'loc_newwest':[0.0],
'loc_abbotsford':[0.0],
'loc_other':[0.0],
'no_basement':[1.0]
import time
from db.propertiesdao import PropertiesDao
from text_mining_service.property_service import PropertyService
from text_mining_service.sklearn_service import SkLearnService
dao = PropertiesDao()
propertyService = PropertyService()
sklearnService = SkLearnService()
rows = dao.getAllPropertiesWithQuery({
"bedrooms": {
"$lt": 3
},
"$or": [{
"bsmt_analyzed": None
}, {
"bsmt_analyzed": 0
}]
})
print("Loaded Records: ", len(rows))
size = len(rows)
count = 0
for property in rows:
property['update'] = False
property = propertyService.verifyBasement(property)
if property['update'] == True:
print(property['_id'], ' - bedrooms:', property['bedrooms'],
' - update basement')
if property['bedrooms'] == 0:
def saveModelData(performance, mean_error, summary=''):
data = {}
data['_id'] = str(ObjectId())
data['model'] = 'XGB Linear Regression'
data['file'] = filename
data['chart'] = chartfilename
data['chart2'] = chartfilename2
data['date'] = datetime.datetime.utcnow()
data['performance'] = performance
data['mean_error'] = mean_error
data['summary'] = summary
dao.saveModel(data)
# #############################################################################
dao = PropertiesDao()
# Get current date and time and create a time stamp
x = datetime.datetime.now()
dtstamp = x.strftime("%Y%m%d")
# Files variables
path = 'data/'
filename = 'reg_xgb_linear_model_' + dtstamp + '.sav'
chartfilename = 'reg_xgb_linear_model_pred_chart_' + dtstamp + '.png'
chartfilename2 = 'reg_xgb_linear_model_importance_chart_' + dtstamp + '.png'
save = False
# Load data
dataset = dao.getDataSetModelPd()
import nltk # Load NLTK
import time
from collections import defaultdict
from db.propertiesdao import PropertiesDao
from text_mining_service.property_service import PropertyService
from text_mining_service.sklearn_service import SkLearnService
# nltk.download()
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
dao = PropertiesDao()
propertyService = PropertyService()
sklearnService = SkLearnService()
print('=====Property Location Update Textmining=====')
rows = dao.getAllPropertiesWithQuery({
"loc_vancouver": 0,
"loc_burnaby": 0,
"loc_richmond": 0,
"loc_surrey": 0,
"loc_newwest": 0,
"loc_abbotsford": 0,
"loc_other": 0
})
print("Records: ", len(rows))
def getProperty(prop_id):
try:
return PropertiesDao().getOnePropertyJson()
except:
return 'No record found!'
def test():
return PropertiesDao().getOnePropertyJson()
def saveModelData(performance, mean_error, summary, bdrmInput = 0):
data = {};
data['_id'] = str(ObjectId())
data['model'] = 'OLS Regression' if bdrmInput == 0 else 'OLS Regression '+ str(bdrmInput)
data['file'] = filename
data['chart'] = chartfilename
data['date'] = datetime.datetime.utcnow()
data['performance'] = performance
data['mean_error'] = mean_error
data['summary'] = summary
dao.saveModel(data)
# #############################################################################
dao = PropertiesDao()
# Get current date and time and create a time stamp
x = datetime.datetime.now()
dtstamp = x.strftime("%Y%m%d")
# Files variables
path = 'data/'
save = False
print('Enter number of bedrooms to analyse and generate a model')
bdrmInput = 0
try:
bdrmInput = int(input())
def __init__(self):
self.dao = PropertiesDao()
self.loadModels()
def saveModelData(performance, mean_error, summary = ''):
data = {};
data['_id'] = str(ObjectId())
data['model'] = 'GradientBoostingRegressor'
data['file'] = filename
data['chart'] = chartfilename
data['chart2'] = chartfilename2
data['chart3'] = chartfilename3
data['date'] = datetime.datetime.utcnow()
data['performance'] = performance
data['mean_error'] = mean_error
data['summary'] = summary
dao.saveModel(data)
# #############################################################################
dao = PropertiesDao()
# Get current date and time and create a time stamp
x = datetime.datetime.now()
dtstamp = x.strftime("%Y%m%d")
# Files variables
path = 'data/'
filename = 'grad_boost_model_'+dtstamp+'.sav'
chartfilename = 'grad_boost_model_pred_chart_'+dtstamp+'.png'
chartfilename2 = 'grad_boost_model_dev_chart_'+dtstamp+'.png'
chartfilename3 = 'grad_boost_model_importance_chart_'+dtstamp+'.png'
save = False
# Load data