class CARTRegressionTree():
def __init__(self, model_name, options=[]):
self.model = None
self.max_leaf_nodes = options[1]
self.criterion = options[3]
self.mode = options[4]
def train_mimic(self, training_data, mimic_env, save_model_dir, log_file):
self.model = DecisionTreeRegressor(max_leaf_nodes=self.max_leaf_nodes,
criterion= self.criterion,
splitter=self.mode)
self.model.fit(training_data[0], training_data[1])
# self.print_tree()
leaves_number = (self.model.tree_.node_count+1)/2
print("Leaves number is {0}".format(leaves_number))
predict_dictionary = {}
predictions = self.model.predict(training_data[0])
for predict_index in range(len(predictions)):
predict_value = predictions[predict_index]
if predict_value in predict_dictionary.keys():
predict_dictionary[predict_value].append(predict_index)
else:
predict_dictionary.update({predict_value:[predict_index]})
return_value_log = mimic_env.get_return(state=list(predict_dictionary.values()))
return_value_log_struct = mimic_env.get_return(state=list(predict_dictionary.values()), apply_structure_cost=True)
return_value_var_reduction = mimic_env.get_return(state=list(predict_dictionary.values()), apply_variance_reduction=True)
mae, rmse = compute_regression_results(predictions=predictions, labels=training_data[1])
# print("Training return:{0} with mae:{1} and rmse:{2}".format(return_value, mae, rmse), file=log_file)
with open(save_model_dir, 'wb') as f:
pickle.dump(obj=self.model, file=f)
return return_value_log, return_value_log_struct, \
return_value_var_reduction, mae, rmse, leaves_number
def test_mimic(self, testing_data, mimic_env, save_model_dir, log_file):
with open(save_model_dir, 'rb') as f:
self.model = pickle.load(file=f)
leaves_number = (self.model.tree_.node_count + 1) / 2
predict_dictionary = {}
predictions = self.model.predict(testing_data[0])
for predict_index in range(len(predictions)):
predict_value = predictions[predict_index]
if predict_value in predict_dictionary.keys():
predict_dictionary[predict_value].append(predict_index)
else:
predict_dictionary.update({predict_value:[predict_index]})
return_value_log = mimic_env.get_return(state=list(predict_dictionary.values()))
return_value_log_struct = mimic_env.get_return(state=list(predict_dictionary.values()), apply_structure_cost=True)
return_value_var_reduction = mimic_env.get_return(state=list(predict_dictionary.values()), apply_variance_reduction=True)
mae, rmse = compute_regression_results(predictions=predictions, labels=testing_data[1])
# print("Testing return:{0} with mae:{1} and rmse:{2}".format(return_value, mae, rmse), file=log_file)
return return_value_log, return_value_log_struct, \
return_value_var_reduction, mae, rmse, leaves_number
class DecisionTreeRegressorImpl():
def __init__(self, criterion='mse', splitter='best', max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features=None, random_state=None, max_leaf_nodes=None, min_impurity_decrease=0.0, min_impurity_split=None, presort=False):
self._hyperparams = {
'criterion': criterion,
'splitter': splitter,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'min_weight_fraction_leaf': min_weight_fraction_leaf,
'max_features': max_features,
'random_state': random_state,
'max_leaf_nodes': max_leaf_nodes,
'min_impurity_decrease': min_impurity_decrease,
'min_impurity_split': min_impurity_split,
'presort': presort}
self._wrapped_model = Op(**self._hyperparams)
def fit(self, X, y=None):
if (y is not None):
self._wrapped_model.fit(X, y)
else:
self._wrapped_model.fit(X)
return self
def predict(self, X):
return self._wrapped_model.predict(X)
def fit(self, X, y, sample_weight=None):
sample_weight = check_sample_weight(y, sample_weight=sample_weight)
assert len(X) == len(y), 'Different lengths of X and y'
X = pandas.DataFrame(X)
y = numpy.array(column_or_1d(y), dtype=int)
assert numpy.all(numpy.in1d(y, [0, 1])), 'Only two-class classification supported'
self.check_params()
self.estimators = []
self.scores = []
n_samples = len(X)
n_inbag = int(self.subsample * len(X))
self.loss = copy.copy(self.loss)
self.loss.fit(X, y, sample_weight=sample_weight)
# preparing for fitting in trees
X = self.get_train_vars(X)
self.n_features = X.shape[1]
X, y = check_arrays(X, y)
X = X.astype(DTYPE)
y_pred = numpy.zeros(len(X), dtype=float)
if self.init_estimator is not None:
y_signed = 2 * y - 1
self.init_estimator.fit(X, y_signed, sample_weight=sample_weight)
y_pred += numpy.ravel(self.init_estimator.predict(X))
for stage in range(self.n_estimators):
# tree creation
tree = DecisionTreeRegressor(
criterion=self.criterion,
splitter=self.splitter,
max_depth=self.max_depth,
min_samples_split=self.min_samples_split,
min_samples_leaf=self.min_samples_leaf,
max_features=self.max_features,
random_state=self.random_state,
max_leaf_nodes=self.max_leaf_nodes)
# tree learning
residual = self.loss.negative_gradient(y_pred)
train_indices = self.random_state.choice(n_samples, size=n_inbag, replace=False)
tree.fit(X[train_indices], residual[train_indices],
sample_weight=sample_weight[train_indices], check_input=False)
# update tree leaves
if self.update_tree:
self.loss.update_tree(tree.tree_, X=X, y=y, y_pred=y_pred, sample_weight=sample_weight,
update_mask=numpy.ones(len(X), dtype=bool), residual=residual)
y_pred += self.learning_rate * tree.predict(X)
self.estimators.append(tree)
self.scores.append(self.loss(y_pred))
return self
def evalOne(parameters):
all_obs = []
all_pred = []
for location in locations:
trainX, testX, trainY, testY = splitDataForXValidation(location, "location", data, all_features, "target")
if "depth" in parameters:
model = DecisionTreeRegressor(max_depth = parameters["depth"], random_state=42)
elif "leaf" in parameters:
model = DecisionTreeRegressor(min_samples_leaf = parameters["leaf"], random_state=42)
elif "max_leaf" in parameters:
model = DecisionTreeRegressor(max_leaf_nodes = parameters["max_leaf"], random_state=42)
model.fit(trainX, trainY)
prediction = model.predict(testX)
all_obs.extend(testY)
all_pred.extend(prediction)
return rmseEval(all_obs, all_pred)[1]
def sklearn_titanic_regression():
from sklearn.tree.tree import DecisionTreeRegressor
from sklearn.preprocessing.label import LabelEncoder
import numpy as np
total_df = pd.read_csv("titanic_clean.csv")
total_df.drop(['cabin', 'boat', 'body', 'index'], axis=1, inplace=True)
total_df.dropna(inplace=True)
for col in total_df.columns.tolist():
if str(total_df[col].dtype) == 'object':
total_df[col] = LabelEncoder().fit_transform(total_df[col])
total_num = total_df.shape[0]
train_df = total_df.iloc[:int(total_num * 0.8)]
test_df = total_df.iloc[int(total_num * 0.8):]
clf = DecisionTreeRegressor()
clf.fit(train_df.drop(['fare'], axis=1), train_df['fare'])
pred = clf.predict(test_df.drop(['fare'], axis=1))
truth = test_df['fare']
mse = np.sum(np.square(pred - truth)) / test_df.shape[0]
print(mse)
max_features="sqrt",
seed=2016,
min_samples=100).fit(X_train, y_train)
y_pred = lrt.predict(X_test)
print("Time taken: %0.3f" % (time.time() - t0))
score = mean_absolute_error(y_test, y_pred)
print("Error: %0.3f" % score)
print("")
print("Sklearn RT")
t0 = time.time()
rt_sklearn = DecisionTreeRegressor(max_depth=7,
max_features="sqrt",
random_state=2016).fit(
X_train, y_train)
y_pred = rt_sklearn.predict(X_test)
print("Time taken: %0.3f" % (time.time() - t0))
score = mean_absolute_error(y_test, y_pred)
print("Error: %0.3f" % score)
print("")
print("Skearn GP")
gp = GaussianProcess(regr="constant",
corr='absolute_exponential',
beta0=None,
storage_mode='full',
verbose=False,
theta0=0.1,
thetaL=None,
thetaU=None,
optimizer='fmin_cobyla',
enc = LabelEncoder()
label_encoder = enc.fit(data['Embarked'])
data['Embarked'] = label_encoder.transform(data['Embarked'])
## predykcja wieku
# TODO: zobaczyć predykcję również tylko po Title
regresor = DecisionTreeRegressor()
X_train_age = data[pd.notnull(data.Age)][['Title', 'SibSp', 'Parch']]
y_train_age = data[pd.notnull(data.Age)][['Age']]
regresor.fit(X_train_age, y_train_age)
# TODO: sprawdzić tą predykcję wieku, działa chyba ok
# data['AgePredicted'] = np.where(pd.isnull(data.Age), regresor.predict(data[['Title', 'SibSp', 'Parch']]), None)
data['Age'] = np.where(pd.isnull(data.Age),
regresor.predict(data[['Title', 'SibSp', 'Parch']]),
data['Age'])
##predykcja poziomu
classifier = DecisionTreeClassifier(max_depth=3, min_samples_leaf=2)
#
X_train_floor = data[pd.notnull(data.Floor)][['Embarked', 'Pclass']]
y_train_floor = data[pd.notnull(data.Floor)]['Floor'].values.astype('int')
classifier.fit(X_train_floor, y_train_floor)
data['Floor'] = np.where(pd.isnull(data.Floor),
classifier.predict(data[['Embarked', 'Pclass']]),
data['Floor'])
##zmiana ceny za bilet
offset = int(0.7 * len(X))
for i in range(10):
X, y = shuffle(boston.data, boston.target)
X_train, y_train = X[:offset], y[:offset]
X_test, y_test = X[offset:], y[offset:]
regressor = GradientBoostingRegressor(max_depth=20, n_estimators=140)
regressor2 = DecisionTreeRegressor(max_depth=6)
regressor3 = LinearRegression()
regressor4 = RandomForestRegressor()
regressor.fit(X_train, y_train)
regressor2.fit(X_train, y_train)
regressor3.fit(X_train, y_train)
regressor4.fit(X_train, y_train)
y_pred = regressor.predict(x)
y_pred2 = regressor2.predict(x)
y_pred3 = regressor3.predict(x)
y_pred4 = regressor4.predict(x)
predictions.append(y_pred)
predictions2.append(y_pred2)
predictions3.append(y_pred3)
predictions4.append(y_pred4)
print "\nPrediction = " + str(y_pred)
print "Prediction = " + str(y_pred2)
print "Prediction = " + str(y_pred3)
print "Prediction = " + str(y_pred4)
print '\n'
print 'Boosting max', np.max(predictions), 'min', np.min(
predictions), 'variance', np.max(predictions) - np.min(predictions)
print 'Decision tree max', np.max(predictions2), 'min', np.min(
def fit(self, X, y, sample_weight=None):
shuffler = Shuffler(X, random_state=self.random_state)
X, y = check_arrays(X, y, dtype=DTYPE, sparse_format="dense", check_ccontiguous=True)
y = column_or_1d(y, warn=True)
n_samples = len(X)
n_inbag = int(self.subsample * n_samples)
sample_weight = check_sample_weight(y, sample_weight=sample_weight).copy()
self.random_state = check_random_state(self.random_state)
# skipping all checks
assert self.update_on in ['all', 'same', 'other', 'random']
y_pred = numpy.zeros(len(y), dtype=float)
self.classifiers = []
self.learning_rates = []
self.loss_values = []
self.loss = copy.copy(self.loss)
self.loss.fit(X, y, sample_weight=sample_weight)
iter_X = shuffler.generate(0.)
prev_smearing = 1
for iteration in range(self.n_estimators):
if iteration % self.recount_step == 0:
if prev_smearing > 0:
iter_smearing = interpolate(self.smearing, iteration, self.n_estimators)
prev_smearing = iter_smearing
iter_X = shuffler.generate(iter_smearing)
iter_X, = check_arrays(iter_X, dtype=DTYPE, sparse_format="dense", check_ccontiguous=True)
y_pred = numpy.zeros(len(y))
y_pred += sum(cl.predict(X) * rate for rate, cl in zip(self.learning_rates, self.classifiers))
self.loss_values.append(self.loss(y, y_pred, sample_weight=sample_weight))
tree = DecisionTreeRegressor(
criterion=self.criterion,
splitter=self.splitter,
max_depth=interpolate(self.max_depth, iteration, self.n_estimators),
min_samples_split=self.min_samples_split,
min_samples_leaf=interpolate(self.min_samples_leaf, iteration, self.n_estimators, use_log=True),
max_features=self.max_features,
random_state=self.random_state)
sample_mask = _random_sample_mask(n_samples, n_inbag, self.random_state)
loss_weight = sample_weight if self.weights_in_loss else numpy.ones(len(sample_weight))
tree_weight = sample_weight if not self.weights_in_loss else numpy.ones(len(sample_weight))
residual = self.loss.negative_gradient(y, y_pred, sample_weight=loss_weight)
tree.fit(numpy.array(iter_X)[sample_mask, :],
residual[sample_mask],
sample_weight=tree_weight[sample_mask], check_input=False)
# update tree leaves
if self.update_tree:
if self.update_on == 'all':
update_mask = numpy.ones(len(sample_mask), dtype=bool)
elif self.update_on == 'same':
update_mask = sample_mask
elif self.update_on == 'other':
update_mask = ~sample_mask
else: # random
update_mask = _random_sample_mask(n_samples, n_inbag, self.random_state)
self.loss.update_terminal_regions(tree.tree_, X=iter_X, y=y, residual=residual, pred=y_pred,
sample_mask=update_mask, sample_weight=sample_weight)
iter_learning_rate = interpolate(self.learning_rate, iteration, self.n_estimators, use_log=True)
y_pred += iter_learning_rate * tree.predict(X)
self.classifiers.append(tree)
self.learning_rates.append(iter_learning_rate)
return self
class FinalEnsembleModel(HCDRDataScorer):
INTERRIM_MODELS = { 'app':'app_model.pkl', 'bureau':'bureau_model.pkl', 'ccb':'ccb_model.pkl', \
'pcb':'pcb_model.pkl', 'pa':'prev_app_model.pkl', 'ip':'ins_pmt_model.pkl'}
# INTERRIM_MODELS = { 'bureau':'bureau_model.pkl' }
score_type = 'train'
def __init__(self, path_to_data_store):
self.path_to_data_store = path_to_data_store
app_data = pd.read_csv(path_to_data_store + '/application_train.csv')
print('Training Set Size', app_data.shape[0])
y_train = app_data['TARGET']
x_train = pd.DataFrame(self.createEnsembleData(app_data[['SK_ID_CURR'
]]),
dtype='float64')
print('Training Set Size After Merging', x_train.shape[0])
x_train.to_csv('ensemble_data.csv')
# self.__regressor__(x_train, y_train)
self.__sv_regressor__(x_train, y_train)
def __regressor__(self, X_train, Y_train):
self.ensemble = DecisionTreeRegressor(random_state=56)
self.ensemble.fit(X_train, Y_train)
print('Ensemble Model Ready')
def __sv_regressor__(self, data, target):
from sklearn.svm.classes import SVR
svr_rbf = SVR(kernel='rbf', C=1e3, gamma=0.1)
svr_rbf.fit(data, target)
self.ensemble = svr_rbf
def __random_forest_regressor__(self, data, target):
from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint
param_distribs = {
'n_estimators': randint(low=1, high=200),
'max_features': randint(low=5, high=8),
}
forest_reg = RandomForestRegressor(random_state=42)
rnd_search = RandomizedSearchCV(forest_reg,
param_distributions=param_distribs,
n_jobs=4,
n_iter=20,
cv=10,
scoring='neg_mean_squared_error',
random_state=42)
# sampling_data = data.drop(['SK_ID_CURR'], axis=1)
rnd_search.fit(data, target)
# cvres = rnd_search.cv_results_
# for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
# print(np.sqrt(-mean_score), params)
self.ensemble = rnd_search.best_estimator_
print('Ensemble Model Ready')
def __curate__(self, data):
data = data.mask(np.isinf(data))
multi_col_na_replace_with_zero = MultiColumnFillNAWithNumericValue(
data.columns, 0)
data = multi_col_na_replace_with_zero.transform(data)
return data
def setScoreType(self, score_type):
self.score_type = score_type
def mean_absolute_percentage_error(self, y_true, y_pred):
y_true, y_pred = np.array(y_true), np.array(y_pred)
return np.mean(np.abs((y_true - y_pred) / y_true)) * 100
def score(self):
test_target = None
test_data = pd.read_csv(self.path_to_data_store +
'/application_test.csv')
sk_id_curr = test_data[['SK_ID_CURR']]
if self.score_type == 'actual':
test_data = self.createEnsembleData(test_data[['SK_ID_CURR']])
else:
test_target = test_data['TARGET']
test_data = self.createEnsembleData(test_data[['SK_ID_CURR']])
score = self.ensemble.predict(test_data)
if test_target is not None:
print(mean_squared_error(test_target.values, score))
output = self.format_output(sk_id_curr, score, test_target)
output.to_csv('submission.csv', index=False)
else:
output = self.format_output(sk_id_curr, score)
output.to_csv('submission.csv', index=False)
def format_output(self, sk_id_curr, preds, target=None):
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler(feature_range=(0, 1))
def ignoreNeg(x):
if x < 0:
return 0
else:
return x
if target is not None:
data = pd.DataFrame()
data['SK_ID_CURR'] = sk_id_curr
data['PREDS'] = scaler.fit_transform([preds])
data['TARGET'] = target
else:
data = pd.DataFrame()
data['SK_ID_CURR'] = sk_id_curr
data['TARGET'] = preds
data['TARGET'] = scaler.fit_transform(data[['TARGET']])
return data
def createEnsembleData(self, app_data):
dataset = None
curr_sk_ids = app_data['SK_ID_CURR'].values
app_data = None
for typ, model_name in self.INTERRIM_MODELS.items():
model = util.load('models/' + model_name)
if typ == 'app':
score_map = model.score(curr_sk_ids, self.score_type)
dataset = pd.DataFrame.from_dict(score_map,
orient='index').reset_index()
dataset.columns = ['SK_ID_CURR', 'DOC', 'REALTY', 'APP']
print('Application data ready .. ')
elif typ == 'bureau':
score_map = model.score(curr_sk_ids)
dataset = pd.DataFrame.from_dict(score_map,
orient='index').reset_index()
dataset.columns = ['SK_ID_CURR', 'BUREAU']
print('Bureau data ready .. ')
elif typ == 'ccb':
score_map = model.score(curr_sk_ids)
dataset = pd.DataFrame.from_dict(score_map,
orient='index').reset_index()
dataset.columns = ['SK_ID_CURR', 'CCB']
print('CCB data ready .. ')
elif typ == 'pcb':
score_map = model.score(curr_sk_ids)
dataset = pd.DataFrame.from_dict(score_map,
orient='index').reset_index()
dataset.columns = ['SK_ID_CURR', 'PCB']
print('PCB data ready .. ')
elif typ == 'pa':
score_map = model.score(curr_sk_ids)
dataset = pd.DataFrame.from_dict(score_map,
orient='index').reset_index()
dataset.columns = ['SK_ID_CURR', 'PREV_APP']
print('PA data ready .. ')
elif typ == 'ip':
score_map = model.score(curr_sk_ids)
dataset = pd.DataFrame.from_dict(score_map,
orient='index').reset_index()
dataset.columns = ['SK_ID_CURR', 'INS_PMT']
print('IP data ready .. ')
if app_data is None:
app_data = dataset
else:
app_data = pd.merge(app_data,
dataset,
how='right',
on=['SK_ID_CURR'])
return self.__curate__(app_data.drop(['SK_ID_CURR'], axis=1))
left_expr = RealNumber(model.tree_.value[current_node, output_idx,
class_idx])
right_expr = left_expr
else:
left_expr = _sym_predict_decision_tree(model,
names,
current_node=left,
output_idx=output_idx,
class_idx=class_idx)
right_expr = _sym_predict_decision_tree(model,
names,
current_node=right,
output_idx=output_idx,
class_idx=class_idx)
return Piecewise(
(left_expr, Symbol(names[model.tree_.feature[current_node]]) <=
model.tree_.threshold[current_node]),
(right_expr, Symbol(names[model.tree_.feature[current_node]]) >
model.tree_.threshold[current_node]),
)
y_pred = model.predict(X)
X_ = pandas.DataFrame(X, columns=map(lambda i: 'x' + str(i), range(10)))
names = list(X_.columns)
expr = _sym_predict_decision_tree(model, names)
for i in range(10):
row = dict(X_.loc[i, :])
assert_almost_equal(y_pred[i], expr.evalf(16, row))
print expr
def fit(self, X, y, sample_weight=None):
shuffler = Shuffler(X, random_state=self.random_state)
X, y = check_arrays(X,
y,
dtype=DTYPE,
sparse_format="dense",
check_ccontiguous=True)
y = column_or_1d(y, warn=True)
n_samples = len(X)
n_inbag = int(self.subsample * n_samples)
sample_weight = check_sample_weight(
y, sample_weight=sample_weight).copy()
self.random_state = check_random_state(self.random_state)
# skipping all checks
assert self.update_on in ['all', 'same', 'other', 'random']
y_pred = numpy.zeros(len(y), dtype=float)
self.classifiers = []
self.learning_rates = []
self.loss_values = []
self.loss = copy.copy(self.loss)
self.loss.fit(X, y, sample_weight=sample_weight)
iter_X = shuffler.generate(0.)
prev_smearing = 1
for iteration in range(self.n_estimators):
if iteration % self.recount_step == 0:
if prev_smearing > 0:
iter_smearing = interpolate(self.smearing, iteration,
self.n_estimators)
prev_smearing = iter_smearing
iter_X = shuffler.generate(iter_smearing)
iter_X, = check_arrays(iter_X,
dtype=DTYPE,
sparse_format="dense",
check_ccontiguous=True)
y_pred = numpy.zeros(len(y))
y_pred += sum(
cl.predict(X) * rate for rate, cl in zip(
self.learning_rates, self.classifiers))
self.loss_values.append(
self.loss(y, y_pred, sample_weight=sample_weight))
tree = DecisionTreeRegressor(
criterion=self.criterion,
splitter=self.splitter,
max_depth=interpolate(self.max_depth, iteration,
self.n_estimators),
min_samples_split=self.min_samples_split,
min_samples_leaf=interpolate(self.min_samples_leaf,
iteration,
self.n_estimators,
use_log=True),
max_features=self.max_features,
random_state=self.random_state)
sample_mask = _random_sample_mask(n_samples, n_inbag,
self.random_state)
loss_weight = sample_weight if self.weights_in_loss else numpy.ones(
len(sample_weight))
tree_weight = sample_weight if not self.weights_in_loss else numpy.ones(
len(sample_weight))
residual = self.loss.negative_gradient(y,
y_pred,
sample_weight=loss_weight)
tree.fit(numpy.array(iter_X)[sample_mask, :],
residual[sample_mask],
sample_weight=tree_weight[sample_mask],
check_input=False)
# update tree leaves
if self.update_tree:
if self.update_on == 'all':
update_mask = numpy.ones(len(sample_mask), dtype=bool)
elif self.update_on == 'same':
update_mask = sample_mask
elif self.update_on == 'other':
update_mask = ~sample_mask
else: # random
update_mask = _random_sample_mask(n_samples, n_inbag,
self.random_state)
self.loss.update_terminal_regions(tree.tree_,
X=iter_X,
y=y,
residual=residual,
pred=y_pred,
sample_mask=update_mask,
sample_weight=sample_weight)
iter_learning_rate = interpolate(self.learning_rate,
iteration,
self.n_estimators,
use_log=True)
y_pred += iter_learning_rate * tree.predict(X)
self.classifiers.append(tree)
self.learning_rates.append(iter_learning_rate)
return self
offset = int(0.7 * len(X))
for i in range(10):
X, y = shuffle(boston.data, boston.target)
X_train, y_train = X[:offset], y[:offset]
X_test, y_test = X[offset:], y[offset:]
regressor = GradientBoostingRegressor(max_depth=20, n_estimators=140)
regressor2 = DecisionTreeRegressor(max_depth=6)
regressor3 = LinearRegression()
regressor4 = RandomForestRegressor()
regressor.fit(X_train, y_train)
regressor2.fit(X_train, y_train)
regressor3.fit(X_train, y_train)
regressor4.fit(X_train, y_train)
y_pred = regressor.predict(x)
y_pred2 = regressor2.predict(x)
y_pred3 = regressor3.predict(x)
y_pred4 = regressor4.predict(x)
predictions.append(y_pred)
predictions2.append(y_pred2)
predictions3.append(y_pred3)
predictions4.append(y_pred4)
print "\nPrediction = " + str(y_pred)
print "Prediction = " + str(y_pred2)
print "Prediction = " + str(y_pred3)
print "Prediction = " + str(y_pred4)
print '\n'
print 'Boosting max', np.max(predictions), 'min', np.min(predictions), 'variance', np.max(predictions) - np.min(predictions)
print 'Decision tree max', np.max(predictions2), 'min', np.min(predictions2), 'variance', np.max(predictions2) - np.min(predictions2)
print 'Random forest max', np.max(predictions4), 'min', np.min(predictions4), 'variance', np.max(predictions4) - np.min(predictions4)
class CARTRegressionTree():
def __init__(self, model_name, options=[]):
self.model = None
self.max_leaf_nodes = options[1]
self.criterion = options[3]
self.mode = options[4]
self.min_samples_leaf = options[6]
# ['max_leaf_nodes', None, 'criterion', 'mse', 'best', 'min_samples_leaf', 10]
def train_mimic(self, training_data, mimic_env, save_model_dir, log_file):
self.model = DecisionTreeRegressor(
max_leaf_nodes=self.max_leaf_nodes,
criterion=self.criterion,
splitter=self.mode,
min_samples_leaf=self.min_samples_leaf,
# max_features = 9
)
self.model.fit(training_data[0], training_data[1])
# self.print_tree()
leaves_number = (self.model.tree_.node_count + 1) / 2
print("Leaves number is {0}".format(leaves_number))
predict_dictionary = {}
predictions = self.model.predict(training_data[0])
for predict_index in range(len(predictions)):
predict_value = predictions[predict_index]
if predict_value in predict_dictionary.keys():
predict_dictionary[predict_value].append(predict_index)
else:
predict_dictionary.update({predict_value: [predict_index]})
return_value_log = mimic_env.get_return(
state=list(predict_dictionary.values()))
return_value_log_struct = mimic_env.get_return(
state=list(predict_dictionary.values()), apply_structure_cost=True)
return_value_var_reduction = mimic_env.get_return(
state=list(predict_dictionary.values()),
apply_variance_reduction=True)
mae, rmse = compute_regression_results(predictions=predictions,
labels=training_data[1])
# print("Training return:{0} with mae:{1} and rmse:{2}".format(return_value, mae, rmse), file=log_file)
with open(save_model_dir, 'wb') as f:
pickle.dump(obj=self.model, file=f)
return return_value_log, return_value_log_struct, \
return_value_var_reduction, mae, rmse, leaves_number
def test_mimic(self, testing_data, mimic_env, save_model_dir, log_file):
with open(save_model_dir, 'rb') as f:
self.model = pickle.load(file=f)
leaves_number = (self.model.tree_.node_count + 1) / 2
predict_dictionary = {}
predictions = self.model.predict(testing_data[0])
for predict_index in range(len(predictions)):
predict_value = predictions[predict_index]
if predict_value in predict_dictionary.keys():
predict_dictionary[predict_value].append(predict_index)
else:
predict_dictionary.update({predict_value: [predict_index]})
return_value_log = mimic_env.get_return(
state=list(predict_dictionary.values()))
return_value_log_struct = mimic_env.get_return(
state=list(predict_dictionary.values()), apply_structure_cost=True)
return_value_var_reduction = mimic_env.get_return(
state=list(predict_dictionary.values()),
apply_variance_reduction=True)
mae, rmse = compute_regression_results(predictions=predictions,
labels=testing_data[1])
# print("Testing return:{0} with mae:{1} and rmse:{2}".format(return_value, mae, rmse), file=log_file)
return return_value_log, return_value_log_struct, \
return_value_var_reduction, mae, rmse, leaves_number
def train_2d_tree(self, training_data, selected_dim=(4, 6)):
training_data = np.stack([
np.asarray(training_data[0])[:, selected_dim[0]],
np.asarray(training_data[0])[:, selected_dim[1]]
],
axis=1)
data_number = 150
self.model.fit(training_data[:data_number],
training_data[1][:data_number])
plot_decision_boundary(input_data=training_data[:data_number],
target_data=training_data[1][:data_number],
tree_model=self.model)
def __del__(self):
pass
def print_tree(self):
n_nodes = self.model.tree_.node_count
children_left = self.model.tree_.children_left
children_right = self.model.tree_.children_right
feature = self.model.tree_.feature
threshold = self.model.tree_.threshold
# The tree structure can be traversed to compute various properties such
# as the depth of each node and whether or not it is a leaf.
node_depth = np.zeros(shape=n_nodes, dtype=np.int64)
is_leaves = np.zeros(shape=n_nodes, dtype=bool)
stack = [(0, -1)] # seed is the root node id and its parent depth
while len(stack) > 0:
node_id, parent_depth = stack.pop()
node_depth[node_id] = parent_depth + 1
# If we have a test node
if (children_left[node_id] != children_right[node_id]):
stack.append((children_left[node_id], parent_depth + 1))
stack.append((children_right[node_id], parent_depth + 1))
else:
is_leaves[node_id] = True
print("The binary tree structure has %s nodes and has "
"the following tree structure:" % n_nodes)
for i in range(n_nodes):
if is_leaves[i]:
print("%snode=%s leaf node." % (node_depth[i] * "\t", i))
else:
print(
"%snode=%s test node: go to node %s if X[:, %s] <= %s else to "
"node %s." % (
node_depth[i] * "\t",
i,
children_left[i],
feature[i],
threshold[i],
children_right[i],
))
@author: TF
'''
import matplotlib.pyplot as plt
import numpy as np
from numpy import *
from sklearn.tree.tree import DecisionTreeRegressor
def plotfigure(X, X_test, y, yp):
plt.figure()
plt.scatter(X, y, c='k', label='data')
plt.plot(X_test, yp, c='r', label='max_depth = 5', linewidth=2)
plt.xlabel('data')
plt.ylabel('target')
plt.title('Decision Tree Regression')
plt.legend()
plt.show()
x = np.linspace(-5, 5, 200)
siny = np.sin(x)
X = mat(x).T
y = siny + np.random.rand(1, len(siny)) * 1.5
y = y.tolist()[0]
clf = DecisionTreeRegressor(max_depth=3)
clf.fit(X, y)
X_test = np.arange(-5.0, 5.0, 0.05)[:, np.newaxis]
yp = clf.predict(X_test)
plotfigure(X, X_test, y, yp)
# load the data
data = {}
columns = []
loadData("/data/york_hour_2013.csv", ["timestamp", "atc"], data, columns)
all_features = deepcopy(columns)
all_features.remove("target")
all_features.remove("location")
output = open(OUTPUT_DATA_FILE, 'w')
output.write("location,observation,prediction\n")
for location in locations:
trainX, testX, trainY, testY = splitDataForXValidation(
location, "location", data, all_features, "target")
model = DecisionTreeRegressor(max_depth=10, random_state=42)
model.fit(trainX, trainY)
prediction = model.predict(testX)
for i in range(0, len(testY)):
output.write(str(location))
output.write(",")
output.write(str(testY[i]))
output.write(",")
output.write(str(prediction[i]))
output.write("\n")
output.close()
class Model(object):
"""
The machine learning component of the tester.
This component stores four different models:
1) A model to decide between different types of events (drags and touches).
2) A model to decide on the starting position for drags.
3) A model to decide on the ending position for drags.
4) A model to decide on the position of the touch.
The input data are all the different known UI elements on the screen from
the training data and whether or not they are visible on the screen.
To acquire this, we first get the stored XML model and record the resource-id
and class. We concatenate them into an array and mark as (1) for visible and (0)
for not visible.
"""
def __init__(self):
self.symbols = {}
self.action_data = None
self.action_labels = None
self.action_classifier = None
self.drag_data = None
self.drag_end_labels = None
self.drag_end_classifier = None
self.drag_start_labels = None
self.drag_start_classifier = None
self.touch_data = None
self.touch_labels = None
self.touch_classifier = None
self.device_info = device.info
def parse_events(self, queue):
symbols = {"randomizer": 0}
events = []
all_data = []
all_results = []
drag_data = []
drag_start_results = []
drag_end_results = []
touch_data = []
touch_results = []
while not queue.empty():
event = queue.get()
events.append(event)
lst = event.state.start.as_list(symbols)
lst[0] = random()
all_data.append(lst)
if event.action.is_drag():
drag_data.append(lst)
all_results.append(DRAG)
start = event.changes.start()
end = event.changes.end()
drag_start_results.append(start.x * start.y)
drag_end_results.append(end.x * end.y)
if event.action.is_touch():
touch_data.append(lst)
all_results.append(TOUCH)
start = event.changes.start()
touch_results.append(start.x * start.y)
if event.action.is_back():
all_results.append(BACK)
data = np.zeros((len(all_data), len(symbols)))
for i, item in enumerate(all_data):
data[i, : len(item)] = item[:]
drags = np.zeros((len(drag_data), len(symbols)))
for i, item in enumerate(drag_data):
drags[i, : len(item)] = item[:]
touches = np.zeros((len(touch_data), len(symbols)))
for i, item in enumerate(touch_data):
touches[i, : len(item)] = item[:]
self.symbols = symbols
self.action_data = data
self.action_labels = np.array(all_results)
self.drag_data = drags
self.drag_start_labels = np.array(drag_start_results)
self.drag_end_labels = np.array(drag_end_results)
self.touch_data = touches
self.touch_labels = np.array(touch_results)
def train(self):
self.action_classifier = DecisionTreeClassifier()
self.action_classifier.fit(self.action_data, self.action_labels)
self.drag_start_classifier = DecisionTreeRegressor()
self.drag_start_classifier.fit(self.drag_data, self.drag_start_labels)
self.drag_end_classifier = DecisionTreeRegressor()
self.drag_end_classifier.fit(self.drag_data, self.drag_end_labels)
self.touch_classifier = DecisionTreeRegressor()
self.touch_classifier.fit(self.touch_data, self.touch_labels)
def predict(self, state):
input = state.as_list(self.symbols, False)
input[0] = random()
action = Action()
type = self.action_classifier.predict(input)
width = self.device_info["displayWidth"]
if type == DRAG:
start = self.drag_start_classifier.predict(input)[0]
end = self.drag_end_classifier.predict(input)[0]
start = Point(start % width, start / width)
end = Point(end % width, end / width)
action.init(ACTION_DRAG, start, end, 0.5)
elif type == TOUCH:
point = self.touch_classifier.predict(input)[0]
point = Point(point % width, point / width)
action.init(ACTION_TOUCH, point.x, point.y)
elif type == BACK:
action.init(ACTION_BACK)
return action
def save(self):
pass