def test_linear(eng):
X = randn(10, 2)
y = fromarray(randn(10, 4).T, engine=eng)
truth = asarray(fit_models(LR, X, y))
betas = LinearRegression().fit(X, y).betas.toarray()
assert allclose(truth, betas)
truth = asarray(fit_models(LR, X, y, fit_intercept=False))
betas = LinearRegression(fit_intercept=False).fit(X, y).betas.toarray()
assert allclose(truth, betas)
def __build_model(self, var_idx, method='forward'):
linear_reg = LinearRegression(gradient_descent=False)
if method == 'forward':
candidate_features = self.__best_features + [var_idx]
elif method == 'backward':
candidate_features = deepcopy(self.__best_features)
candidate_features.remove(var_idx)
X = self.X[:, candidate_features]
linear_reg.fit(X, self.y)
y_preds = [linear_reg.predict(x) for x in X]
return calculate_r2(self.y, y_preds)
def test_predict_and_score(eng): X = randn(10, 2) y = fromarray(randn(10, 4).T, engine=eng) model = LinearRegression().fit(X, y) yhat = model.predict(X).toarray() rsq = model.score(X, y).toarray() truth = hstack([yhat, rsq[:, newaxis]]) result = model.predict_and_score(X, y).toarray() assert allclose(truth, result)
def test_predict_and_score(eng):
X = randn(10, 2)
y = fromarray(randn(10, 4).T, engine=eng)
model = LinearRegression().fit(X, y)
yhat = model.predict(X).toarray()
rsq = model.score(X, y).toarray()
truth = hstack([yhat, rsq[:, newaxis]])
result = model.predict_and_score(X, y).toarray()
assert allclose(truth, result)
def test_predict(eng):
X = randn(10, 2)
y = fromarray(randn(10, 4).T, engine=eng)
truth = asarray(predict_models(LR, X, y))
predictions = LinearRegression().fit(X, y).predict(X).toarray()
assert allclose(truth, predictions)
def test_score(eng):
X = randn(10, 2)
y = fromarray(randn(10, 4).T, engine=eng)
truth = asarray(score_models(LR, X, y))
scores = LinearRegression().fit(X, y).score(X, y).toarray()
assert allclose(truth, scores)
def single_linear_regression_model(single_linear_regression_data):
linear_regression_model = LinearRegression(
independent_vars=single_linear_regression_data["independent_vars"],
dependent_var=single_linear_regression_data["dependent_var"],
iterations=10000,
learning_rate=0.001,
train_split=0.7,
seed=123,
)
return linear_regression_model
def test_betas_and_scores(eng):
X = randn(10, 2)
y = fromarray(randn(10, 4).T, engine=eng)
true_betas = asarray(fit_models(LR, X, y))
true_scores = asarray(score_models(LR, X, y))
truth = hstack([true_betas, true_scores[:, newaxis]])
result = LinearRegression().fit(X, y).betas_and_scores.toarray()
assert allclose(truth, result)
def build_tree(x_data, y_data):
feature, split_val = best_split(x_data, y_data)
node = TreeNode(feature, split_val)
if feature is None:
node.model = LinearRegression()
node.model.fit(x_data, y_data)
else:
idx = (x_data[:, feature] <= split_val)
node.left = build_tree(x_data[idx], y_data[idx])
node.right = build_tree(x_data[~idx], y_data[~idx])
return node
def build_models(self, df):
self.n, self.p = df.shape
performances = []
for k in range(1, self.p):
for var_combo in itertools.combinations(df.columns[:-1], k):
linear_reg = LinearRegression()
X = np.asarray(df[list(var_combo)])
self.y = np.asarray(df.iloc[:, -1])
linear_reg.fit(X, self.y)
y_preds = [linear_reg.predict(x) for x in X]
adj_r2, aic, bic, r2, rss = self.__calculate_criterions(
y_preds, k)
performance = [var_combo, k, aic, bic, rss, r2, adj_r2]
performances.append(performance)
col_names = [
'subset', 'num_of_variables', 'aic', 'bic', 'rss', 'r2',
'adj_r2'
]
self.models_summary = pd.DataFrame(performances,
columns=col_names)
self.__visualize_best_subset_performance()
def linear_regression (self):
data_preprocessor = DataPreProcessor(self.df_variable, self.pd_variable) # define preprocess.DataPreProcessor instance
data_preprocessor_for_task = DataPreProcessorForTask(data_preprocessor) # perform preprocessing using preprocess.DataPreProcessorForTask using preprocess.DataPreProcessor instance
data_preprocessor_for_task.preprocess_for_linear_regression() # perform preprocessing for Linear Regression
regression_pd_variable = data_preprocessor.pd_variable
regression_df_variable = data_preprocessor.df_variable
linear_regression = LinearRegression(regression_df_variable) # define instance of regression.LinearRegression on preproccessed DF
linear_regression.regression() # run regression
coefficients = linear_regression.coefficients # regression retrieve its coefficients result
intercept = linear_regression.intercept # regression retrieve its intercept result
liner_equation = linear_regression.liner_equation # regression retrieve its linear equation result
mean_squared_error = linear_regression.mean_squared_error # regression retrieve its MSE result
messagebox.showinfo("liner_equation", liner_equation) # show linear equation on message box
messagebox.showinfo("mean_squared_error", mean_squared_error) # show regression MSE on message box
print (coefficients)
print (intercept)
print (liner_equation)
print (mean_squared_error)
def run(self):
x, y = self.__readData()
model = LinearRegression()
model.fit(x, y)
y_predicted = [model.sumForRow(row) for row in x]
# this plot is just to make sure we get values of a linear function
plt.scatter(y, y_predicted, c='r')
plt.show()
mean_error = model.error(y_predicted, y)
return mean_error
def regression():
#LinearRegression
resultString = ''
input1 = ''
input2 = ''
output1 = ''
liner_regression = LinearRegression()
print("okay")
if request.method == "POST":
input1 = request.form.get('input1')
print("input1", input1)
input_x_list = liner_regression.regression_string_2_list(input1)
print("input_x_list", input_x_list)
input2 = request.form.get('input2')
print("input2", input2)
input_y_list = liner_regression.regression_string_2_list(input2)
print("input_y_list", input_y_list)
liner_regression.model(input_x_list, input_y_list)
output1 = request.form.get('output1')
print("output1", output1)
predict_x_list = liner_regression.regression_string_2_list(output1)
print("predict_x_list", predict_x_list)
predict_y_list = liner_regression.predict(predict_x_list)
print("predict_y_list", predict_y_list)
resultString = ''
for i in range(len(predict_y_list)):
resultString += str(predict_y_list[i]) + ', '
print("resultString", resultString)
return render_template('regression.html', title = 'Regression', resultString = resultString, input1 = input1,input2 = input2, output1 =output1) # customize the title
return render_template('regression.html', title = 'Regression', resultString = resultString, input1 = input1,input2 = input2, output1 =output1) # customize the title
def main():
X, y = make_regression(n_samples=100, n_features=1, noise=20)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
n_samples, n_features = np.shape(X)
model = LinearRegression(n_iterations=100)
model.fit(X_train, y_train)
# Training error plot
n = len(model.training_errors)
training, = plt.plot(range(n), model.training_errors, label="Training Error")
plt.legend(handles=[training])
plt.title("Error Plot")
plt.ylabel('Mean Squared Error')
plt.xlabel('Iterations')
plt.show()
y_pred = model.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print ("Mean squared error: %s" % (mse))
y_pred_line = model.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
plt.plot(366 * X, y_pred_line, color='b', linewidth=2, label="Prediction")
plt.suptitle("Linear Regression")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.show()
def main():
X, y = make_regression(n_samples=100, n_features=1, noise=20)
x_train, x_test, y_train, y_test = DataManipulation().train_test_split(
X, y, test_size=0.4)
n_samples, n_features = X.shape
model = LinearRegression()
model.fit(x_train, y_train)
n = len(model.errors)
training = plt.plot(range(n), model.errors, label='Training Errors')
plt.title('Error plot')
plt.xlabel('Iteration')
plt.ylabel('Mean Squared Error')
plt.show()
y_pred = model.predict(x_test)
y_pred_line = model.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
# Plot the results
m1 = plt.scatter(366 * x_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * x_test, y_test, color=cmap(0.5), s=10)
plt.plot(366 * X,
y_pred_line,
color='black',
linewidth=2,
label="Prediction")
plt.suptitle("Linear Regression")
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.show()
data = pd.read_csv('house.csv')
max = data['size'].max()
data['size'] = data['size'].apply(lambda x: x / max)
bedroomMax = data['bedroom'].max()
data['bedroom'] = data['bedroom'].apply(lambda x: x / bedroomMax)
size = data['size'].values
bedroom = data['bedroom'].values
price = data['price'].values
X = np.array([np.ones(len(size)), size, bedroom]).T
Y = np.array(price)
regression = LinearRegression(alpha=0.006, iteration=1000, feature_count=2)
regression.fit(X, Y)
regression.plot()
while (True):
size = int(input("Enter size of house:"))
bedroom = int(input("Enter number of bedroom:"))
print("price:", int(regression.predict(np.array([1, size, bedroom]))))
def __get_full_model_r2(self):
linear_reg = LinearRegression(gradient_descent=False)
linear_reg.fit(self.X, self.y)
y_preds = [linear_reg.predict(x) for x in self.X]
return calculate_r2(self.y, y_preds)
max = data['floor'].max()
data['floor'] = data['floor'].apply(lambda x: x / max)
max = data['top_floor'].max()
data['top_floor'] = data['top_floor'].apply(lambda x: x / max)
price = data['price'].values
X = np.array([np.ones(len(price)), data['size'].values, data['room'].values, data['year'].values, data['floor'].values, data['top_floor'].values]).T
Y = np.array(price)
regression = LinearRegression(alpha=0.000001, iteration=300, feature_count=5)
regression.fit(X, Y)
regression.plot()
print("price:", "13800000")
print("price:", int(regression.predict(np.array([1, 45,2,1977,5,5]))))
print()
print("price:", "15333333")
print("price:", int(regression.predict(np.array([1, 40,2,1967,1,4]))))
# while (True):
# size = int(input("Enter size of house:"))
# bedroom = int(input("Enter number of bedroom:"))
# print("price:", int(regression.predict(np.array([1, size, bedroom]))))
# Fake training set for property rental with feature scaling
training_set_rent = np.array([
[(1 - 2.5) / 5, (500 - 500) / 1000, (24000 - 50000) / 100000],
[(1 - 2.5) / 5, (1000 - 500) / 1000, (22000 - 50000) / 100000],
[(1 - 2.5) / 5, (1000 - 500) / 1000, (22000 - 50000) / 100000],
[(1 - 2.5) / 5, (1000 - 500) / 1000, (22000 - 50000) / 100000],
[(2 - 2.5) / 5, (500 - 500) / 1000, (32000 - 50000) / 100000],
[(2 - 2.5) / 5, (1000 - 500) / 1000, (29000 - 50000) / 100000],
[(3 - 2.5) / 5, (500 - 500) / 1000, (40000 - 50000) / 100000],
[(1 - 2.5) / 5, (500 - 500) / 1000, (24000 - 50000) / 100000],
[(1 - 2.5) / 5, (1000 - 500) / 1000, (22000 - 50000) / 100000],
[(2 - 2.5) / 5, (500 - 500) / 1000, (32000 - 50000) / 100000],
[(2 - 2.5) / 5, (1000 - 500) / 1000, (29000 - 50000) / 100000],
[(3 - 2.5) / 5, (500 - 500) / 1000, (40000 - 50000) / 100000],
])
# Create and train LR with gradient
linear = LinearRegression(training_set_rent)
linear.train_gradient()
linear.show_info()
# Try to predict something
print(linear.hypothesis([(1 - 2.5) / 5, (1000 - 500) / 1000]) * 100000 + 50000)
# Create and train LR with normal equation
linear_with_normal = LinearRegression(training_set_rent)
linear_with_normal.normal_equation()
linear_with_normal.show_info()
def main():
#from sklearn.datasets import load_boston
#boston = load_boston()
#print(boston.data.shape)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG if DEBUG else logging.INFO)
dataFile = "data/housing.data"
col_names = ["crim", "zn", "indus", "chas", "nox", "rm", "age", "dis", "rad", "tax", "ptratio", "b", "lstat", "medv"]
train_df = pd.read_csv(dataFile, names = col_names, delim_whitespace = True)
test_df = train_df.iloc[::7, :]
train_df.drop(train_df.index[::7], inplace=True)
train_df_features = train_df.iloc[:, :-1]
train_df_targets = train_df.iloc[:, -1]
test_df_features = test_df.iloc[:, :-1]
test_df_targets = test_df.iloc[:, -1]
# Data analysis
print("Data analysis:")
print("No. of attributes: ", len(train_df.iloc[0]))
print("No. of features usable for classifcation: ", len(train_df.iloc[0])-1)
print("Size of training data: ", len(train_df))
print("Size of testing data: ", len(test_df))
print("Histogram of attributes will be shown at the end of generating all results")
print("\nPearson correlations:")
target_col = col_names[-1]
for col in col_names:
if col.lower() == 'chas': # categorical. Also, see dtypes
continue
print("Correlation of %s with target(%s): %f" % (col, target_col, train_df[[col, target_col]].corr(method='pearson').iloc[0,1]))
normalizer = DataFrameStdNormalizer(train_df_features)
train_df_features_normalized = normalizer.get_normalized_data(train_df_features)
test_df_features_normalized = normalizer.get_normalized_data(test_df_features)
print("\n*********************Linear Regression*******************")
regmodel = LinearRegression()
eval = ModelEvaluator(regmodel)
regmodel.train(train_df_features_normalized, train_df_targets)
trainingError = eval.mean_squared_error(train_df_features_normalized, train_df_targets)
print("Mean squared error on training data: %f" % trainingError)
print("Mean squared error on test data: %f" % eval.mean_squared_error(test_df_features_normalized, test_df_targets))
print("\n***********Ridge regression with lambda 0.01m 0.1, 1.0***************")
for lambdaval in (0.01, 0.1, 1.0):
regmodel = RidgeRegression(lambdaval)
eval = ModelEvaluator(regmodel)
regmodel.train(train_df_features_normalized, train_df_targets)
trainingError = eval.mean_squared_error(train_df_features_normalized, train_df_targets)
testingError = eval.mean_squared_error(test_df_features_normalized, test_df_targets)
print("Ridge regression model with lambda = %f" % lambdaval)
print("Mean squared error on training data = %f" % trainingError)
print("Mean squared error on test data = %f" % testingError)
print("")
print("\n*********************Cross Validation*******************")
lambdaval = float(10.0)
# Shuffle data
shuffled_train_df = train_df.reindex(np.random.permutation(train_df.index))
shuffled_train_df_features = train_df.iloc[:, :-1]
shuffled_train_df_targets = train_df.iloc[:, -1]
shuffled_train_df_features_normalized = (DataFrameStdNormalizer(shuffled_train_df_features)).get_normalized_data(shuffled_train_df_features)
lambda_error_map = {}
for i in range(0,6):
lambdaval = float(10.0) / (10**i)
# cross validation
mean_cv_error = 0
regmodel = RidgeRegression(lambdaval)
eval = ModelEvaluator(regmodel)
for i in range(0,10):
chunksize = len(train_df)/10
test_df_cv = None
train_df_cv_targets = None
test_df_cv = None
test_df_cv_targets = None
test_df_cv = shuffled_train_df_features_normalized.iloc[i*chunksize:i*chunksize+chunksize]
test_df_cv_targets = shuffled_train_df_targets.iloc[i*chunksize:i*chunksize+chunksize]
train_df_cv = shuffled_train_df_features_normalized.drop(shuffled_train_df_features_normalized.index[i*chunksize:i*chunksize+chunksize])
train_df_cv_targets = shuffled_train_df_targets.drop(shuffled_train_df_targets.index[i*chunksize:i*chunksize+chunksize])
regmodel.train(train_df_cv, train_df_cv_targets)
#print(eval.mean_squared_error(test_df_cv, test_df_cv_targets))
mean_cv_error += eval.mean_squared_error(test_df_cv, test_df_cv_targets)
mean_cv_error /= 10
print("MSE for lambda %f = %f" % (lambdaval, mean_cv_error))
lambda_error_map[lambdaval] = mean_cv_error
lambdabest = min(lambda_error_map, key=lambda_error_map.get)
print("Lowest MSE for lambda = %f" % lambdabest)
regmodel = RidgeRegression(lambdabest)
regmodel.train(train_df_features_normalized, train_df_targets)
eval = ModelEvaluator(regmodel)
test_meansquarederror = eval.mean_squared_error(test_df_features_normalized, test_df_targets)
print("Test error for model with lambda %f = %f" % (lambdabest, test_meansquarederror))
print("")
print("\n*********************Feature Selection*******************")
print("*********************i. Max correlation*******************")
target_col = col_names[-1]
corr = {}
for col in col_names:
if col.lower() == 'chas': # categorical. Also, see dtypes
continue
corr[col] = abs(train_df[[col, target_col]].corr(method='pearson').iloc[0,1])
maxcorrcols = heapq.nlargest(5, corr, key=corr.get)[1:]
print("Selecting the following coluns with max correlation: ")
print(maxcorrcols)
train_df_features_normalized_maxcorr = train_df_features[maxcorrcols]
regmodel = LinearRegression()
regmodel.train(train_df_features_normalized_maxcorr, train_df_targets)
eval = ModelEvaluator(regmodel)
trainingError = eval.mean_squared_error(train_df_features[maxcorrcols], train_df_targets)
print("Mean squared error on training data: %f" % trainingError)
print("Mean squared error on test data: %f" % eval.mean_squared_error(test_df_features[maxcorrcols], test_df_targets))
print("*******************ii. Max correlation with residue*****************")
residue = train_df_targets.copy(deep=True)
cols = []
regmodel = LinearRegression()
eval = ModelEvaluator(regmodel)
for i in range(0, 4):
corr = {}
for col in col_names:
if col.lower() in ('medv', 'chas') or col in cols: # categorical. Also, see dtypes
continue
# corr[col] = train_df[[col]].corrwith(residue).iloc[0]
corr[col] = abs(pd.concat([train_df[[col]], residue], axis = 1).corr(method='pearson').iloc[0,1])
maxcorrcol = max(corr, key=corr.get)
cols.append(maxcorrcol)
print("Taking cols: %s" % maxcorrcol)
regmodel.train(train_df_features[cols], train_df_targets)
for i in range(0,len(residue)):
residue.at[residue.index[i]] = train_df_targets.iloc[i] - regmodel.predict(train_df_features[cols].iloc[i])
#trainingError = eval.mean_squared_error(train_df_features_normalized, train_df_targets)
#print("Mean squared error on training data: %f" % trainingError)
#print(cols)
print("Mean squared error on train data: %f" % eval.mean_squared_error(train_df_features[cols], train_df_targets))
print("Mean squared error on test data: %f" % eval.mean_squared_error(test_df_features[cols], test_df_targets))
print("*********************iii. All 4 feature combinations*******************")
bestcols = None
besttrainmse = 999999
regmodel = LinearRegression()
eval = ModelEvaluator(regmodel)
for cols in list(list(x) for x in itertools.combinations(train_df_features_normalized.columns, 4)):
regmodel.train(train_df_features_normalized[cols], train_df_targets)
mse_train = eval.mean_squared_error(train_df_features_normalized[cols], train_df_targets)
#print("Mean squared error on train data: %f" % )
#print("Mean squared error on test data: %f" % eval.mean_squared_error(train_df_features_normalized[cols], train_df_targets))
if mse_train < besttrainmse:
bestcols = cols
besttrainmse = mse_train
print("Best training MSE = %f for columns:" % besttrainmse)
print(bestcols)
regmodel.train(train_df_features_normalized[bestcols], train_df_targets)
print("Testing MSE of this model: %f" % eval.mean_squared_error(test_df_features_normalized[cols], test_df_targets))
print("\n*********************Feature Expansion*******************")
df_train_featuregen = train_df_features_normalized.copy(deep=True)
df_test_featuregen = test_df_features_normalized.copy(deep=True)
#i = 0
for cols in list(list(x) for x in itertools.combinations(train_df_features_normalized.columns, 2)) + [[col,col] for col in train_df_features_normalized.columns]:
#i += 1
#print("Gen %d: %s" % (i,cols[0]+cols[1]))
#df_train_featuregen[cols[0]+cols[1]] = df_train_featuregen.apply(lambda x: [[x[cols[0]], x[cols[1]]]], axis=1)
df_train_featuregen[cols[0]+cols[1]] = df_train_featuregen[cols[0]]*df_train_featuregen[cols[1]]
df_test_featuregen[cols[0]+cols[1]] = df_test_featuregen[cols[0]]*df_test_featuregen[cols[1]]
#df_test_featuregen[cols[0]+cols[1]] = df_test_featuregen.apply(lambda x: [[x[cols[0]], x[cols[1]]]], axis=1)
regmodel = LinearRegression()
regmodel.train(df_train_featuregen, train_df_targets)
eval = ModelEvaluator(regmodel)
trainingError = eval.mean_squared_error(df_train_featuregen, train_df_targets)
print("Mean squared error on training data: %f" % trainingError)
print("Mean squared error on test data: %f" % eval.mean_squared_error(df_test_featuregen, test_df_targets))
print("\n******************************** Showing histogram of attributes********************************")
Histogrammer.plot_histgram_of_features(train_df, 3, 5)
print("\nClose window to terminate")
#plt.show(block=False) #.draw()
#plt.pause(0.001)
#raw_input("Press enter to continue")
plt.show()
return
def test_simple_linear_regression(self):
self._run_simple_linear_regression_for_model(LinearRegression())
def test_linear_regression():
# define sample data
xs = np.linspace(0, 2, 50).astype(np.float32)[:, np.newaxis]
noise = np.random.normal(0, 0.5, xs.shape).astype(np.float32)
ys = xs + noise
# define figure
fig = plt.figure('linear regression')
# show the sample data
ax = fig.add_subplot(1, 1, 1, title='linear regression')
ax.scatter(xs, ys, c='r', label='sample points')
# show real curve
_ys = xs
ax.plot(xs, _ys, 'g', label='real curve')
# define linear regression operation
sgd_optimizer = GradientDescentOptimizer(0.008, 1)
linear_regression = LinearRegression(xs, ys, sgd_optimizer)
# start regression
ax.plot(xs, linear_regression.predict(xs), 'b', label='regression curve')
plt.legend(loc='upper left')
plt.ion()
for step in range(50):
plt.pause(0.01)
linear_regression.regress_once()
ax.lines.pop()
ax.plot(xs,
linear_regression.predict(xs),
'b',
label='regression curve')
print('step {}: loss = {}'.format(step,
linear_regression.compute_loss()))
print('\n', '-' * 20, 'optimize end', '-' * 20, '\n')
print('weights : \n{}'.format(linear_regression.weights))
print('loss : \n{}'.format(linear_regression.compute_loss()))
print('-' * 50, '\n')
print('compute directly: ')
linear_regression.weights = LinearRegression.compute_weights(xs, ys)
print('weights: \n{}'.format(linear_regression.weights))
print('loss: \n{}'.format(linear_regression.compute_loss()))
print('-' * 50, '\n')
print('real value: ')
linear_regression.weights = np.array([1, 0], dtype=np.float32)[:,
np.newaxis]
print('weights: \n{}'.format(
np.array([1, 0], dtype=np.float32)[:, np.newaxis]))
print('loss : \n{}'.format(linear_regression.compute_loss()))
print('-' * 50, '\n')
ax.plot(xs, linear_regression.predict(xs), 'g', label='real curve')
plt.ioff()
plt.savefig('linear regression.png')
plt.show()
""" @author Victor I. Afolabi A.I. Engineer & Software developer [email protected] Created on 26 August, 2017 @ 9:33 PM. Copyright (c) 2017. victor. All rights reserved. """ # Create a LinearRegression object from regression import LinearRegression import numpy as np data = np.genfromtxt('data.csv', delimiter=',') num_iter = 1000 clf = LinearRegression(learning_rate=1e-4) clf.fit(data=data, num_iter=num_iter) print('After {:,} iterations. m = {:.2f} and b = {:.2f}'.format(num_iter, clf.m, clf.b))
def linear_solve(x_data, y_data):
model = LinearRegression()
model.fit(x_data, y_data)
return model.predict(x_data)
import pandas as pd
import numpy as np
from regression import LinearRegression
df = pd.read_csv("/Users/yliang/data/trunk1/spark/assembly/target/tmp/LinearRegressionSuite/datasetWithDenseFeature2/part-00000", header = None)
X = np.array(df[df.columns[1:3]])
y = np.array(df[df.columns[0]])
lir = LinearRegression(fit_intercept=True, alpha=2.3, max_iter=100, tol=1e-06, standardization=False,
lower_bound=[-np.inf, 6.0, -np.inf], upper_bound=[0.0, 10.0, np.inf])
lir.fit(X, y)
print("coefficients = " + str(lir.coef_))
print("intercept = " + str(lir.intercept_))
def __test_bootstrap_fit():
# A small implementation of a test case
from regression import LinearRegression
N_bs = 1000
# Initial values
n = 200
noise = 0.2
np.random.seed(1234)
test_percent = 0.35
# Sets up random matrices
x = np.random.rand(n, 1)
def func_excact(_x): return 2*_x*_x + np.exp(-2*_x) + noise * \
np.random.randn(_x.shape[0], _x.shape[1])
y = func_excact(x)
def design_matrix(_x):
return np.c_[np.ones(_x.shape), _x, _x*_x]
# Sets up design matrix
X = design_matrix(x)
# Performs regression
reg = LinearRegression()
reg.fit(X, y)
y = y.ravel()
y_predict = reg.predict(X).ravel()
print("Regular linear regression")
print("R2: {:-20.16f}".format(reg.score(y_predict, y)))
print("MSE: {:-20.16f}".format(metrics.mse(y, y_predict)))
print("Beta: ", reg.coef_.ravel())
print("var(Beta): ", reg.coef_var.ravel())
print("")
# Performs a bootstrap
print("Bootstrapping")
bs_reg = BootstrapRegression(x, y, LinearRegression, design_matrix)
bs_reg.bootstrap(N_bs, test_percent=test_percent)
print("R2: {:-20.16f}".format(bs_reg.R2))
print("MSE: {:-20.16f}".format(bs_reg.MSE))
print("Bias^2:{:-20.16f}".format(bs_reg.bias))
print("Var(y):{:-20.16f}".format(bs_reg.var))
print("Beta: ", bs_reg.coef_.ravel())
print("var(Beta): ", bs_reg.coef_var.ravel())
print("MSE = Bias^2 + Var(y) = ")
print("{} = {} + {} = {}".format(bs_reg.MSE, bs_reg.bias, bs_reg.var,
bs_reg.bias + bs_reg.var))
print("Diff: {}".format(abs(bs_reg.bias + bs_reg.var - bs_reg.MSE)))
import matplotlib.pyplot as plt
plt.plot(x.ravel(), y, "o", label="Data")
plt.plot(x.ravel(), y_predict, "o",
label=r"Pred, R^2={:.4f}".format(reg.score(y_predict, y)))
print (bs_reg.y_pred.shape, bs_reg.y_pred_var.shape)
plt.errorbar(bs_reg.x_pred_test, bs_reg.y_pred,
yerr=np.sqrt(bs_reg.y_pred_var), fmt="o",
label=r"Bootstrap Prediction, $R^2={:.4f}$".format(bs_reg.R2))
plt.xlabel(r"$x$")
plt.ylabel(r"$y$")
plt.title(r"$2x^2 + \sigma^2$")
plt.legend()
plt.show()
def __test_cross_validation_methods():
# A small implementation of a test case
from regression import LinearRegression
import matplotlib.pyplot as plt
# Initial values
n = 100
N_bs = 1000
k_splits = 4
test_percent = 0.2
noise = 0.3
np.random.seed(1234)
# Sets up random matrices
x = np.random.rand(n, 1)
def func_excact(_x): return 2*_x*_x + np.exp(-2*_x) + noise * \
np.random.randn(_x.shape[0], _x.shape[1])
y = func_excact(x)
def design_matrix(_x):
return np.c_[np.ones(_x.shape), _x, _x * _x]
# Sets up design matrix
X = design_matrix(x)
# Performs regression
reg = LinearRegression()
reg.fit(X, y)
y = y.ravel()
y_predict = reg.predict(X).ravel()
print("Regular linear regression")
print("R2: {:-20.16f}".format(reg.score(y, y_predict)))
print("MSE: {:-20.16f}".format(metrics.mse(y, y_predict)))
# print (metrics.bias(y, y_predict))
print("Bias^2:{:-20.16f}".format(metrics.bias2(y, y_predict)))
# Small plotter
import matplotlib.pyplot as plt
plt.plot(x, y, "o", label="data")
plt.plot(x,
y_predict,
"o",
label=r"Pred, $R^2={:.4f}$".format(reg.score(y, y_predict)))
print("k-fold Cross Validation")
kfcv = kFoldCrossValidation(x, y, LinearRegression, design_matrix)
kfcv.cross_validate(k_splits=k_fold_size, test_percent=test_percent)
print("R2: {:-20.16f}".format(kfcv.R2))
print("MSE: {:-20.16f}".format(kfcv.MSE))
print("Bias^2:{:-20.16f}".format(kfcv.bias))
print("Var(y):{:-20.16f}".format(kfcv.var))
print("MSE = Bias^2 + Var(y) = ")
print("{} = {} + {} = {}".format(kfcv.MSE, kfcv.bias, kfcv.var,
kfcv.bias + kfcv.var))
print("Diff: {}".format(abs(kfcv.bias + kfcv.var - kfcv.MSE)))
plt.errorbar(kfcv.x_pred_test,
kfcv.y_pred,
yerr=np.sqrt(kfcv.y_pred_var),
fmt="o",
label=r"k-fold CV, $R^2={:.4f}$".format(kfcv.R2))
print("kk Cross Validation")
kkcv = kkFoldCrossValidation(x, y, LinearRegression, design_matrix)
kkcv.cross_validate(k_splits=k_fold_size, test_percent=test_percent)
print("R2: {:-20.16f}".format(kkcv.R2))
print("MSE: {:-20.16f}".format(kkcv.MSE))
print("Bias^2:{:-20.16f}".format(kkcv.bias))
print("Var(y):{:-20.16f}".format(kkcv.var))
print("MSE = Bias^2 + Var(y) = ")
print("{} = {} + {} = {}".format(kkcv.MSE, kkcv.bias, kkcv.var,
kkcv.bias + kkcv.var))
print("Diff: {}".format(abs(kkcv.bias + kkcv.var - kkcv.MSE)))
plt.errorbar(kkcv.x_pred_test.ravel(),
kkcv.y_pred.ravel(),
yerr=np.sqrt(kkcv.y_pred_var.ravel()),
fmt="o",
label=r"kk-fold CV, $R^2={:.4f}$".format(kkcv.R2))
print("Monte Carlo Cross Validation")
mccv = MCCrossValidation(x, y, LinearRegression, design_matrix)
mccv.cross_validate(N_bs, k_splits=k_fold_size, test_percent=test_percent)
print("R2: {:-20.16f}".format(mccv.R2))
print("MSE: {:-20.16f}".format(mccv.MSE))
print("Bias^2:{:-20.16f}".format(mccv.bias))
print("Var(y):{:-20.16f}".format(mccv.var))
print("MSE = Bias^2 + Var(y) = ")
print("{} = {} + {} = {}".format(mccv.MSE, mccv.bias, mccv.var,
mccv.bias + mccv.var))
print("Diff: {}".format(abs(mccv.bias + mccv.var - mccv.MSE)))
print("\nCross Validation methods tested.")
plt.errorbar(mccv.x_pred_test,
mccv.y_pred,
yerr=np.sqrt(mccv.y_pred_var),
fmt="o",
label=r"MC CV, $R^2={:.4f}$".format(mccv.R2))
plt.xlabel(r"$x$")
plt.ylabel(r"$y$")
plt.title(r"$y=2x^2$")
plt.legend()
plt.show()
action="store_true")
parser.add_argument(
"--loo",
help=
"Calculate leave-one-out error. Will have an adverse effect on run-time.",
action="store_true")
return parser.parse_args()
if __name__ == '__main__':
args = parseArguments()
settings = dict()
if args.noPlot:
settings['plot'] = False
else:
settings['plot'] = True
if args.loo:
settings['loo'] = True
else:
settings['loo'] = False
if args.type == 'linear':
# Linear Regression
LinearRegression(calculate_error=settings.get('loo'),
plot_now=settings.get('plot'))
elif args.type == 'logistic':
# Logistic Regression
LogisticRegression(calculate_error=settings.get('loo'),
plot_now=settings.get('plot'))